You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafu.apache.org by wv...@apache.org on 2014/03/04 08:09:19 UTC
[01/19] DATAFU-27 Migrate build system to Gradle
Repository: incubator-datafu
Updated Branches:
refs/heads/master 4aa2ef2a4 -> 778bef1eb
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/urls/UserAgentTest.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/urls/UserAgentTest.java b/test/pig/datafu/test/pig/urls/UserAgentTest.java
deleted file mode 100644
index 82515ad..0000000
--- a/test/pig/datafu/test/pig/urls/UserAgentTest.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.urls;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.test.pig.PigTests;
-
-public class UserAgentTest extends PigTests
-{
-
- /**
- register $JAR_PATH
-
- define UserAgentClassify datafu.pig.urls.UserAgentClassify();
-
- data = load 'input' as (usr_agent:chararray);
- data_out = foreach data generate UserAgentClassify(usr_agent) as class;
- --describe data_out;
- store data_out into 'output';
- */
- @Multiline private String userAgentTest;
-
- @Test
- public void userAgentTest() throws Exception
- {
- PigTest test = createPigTestFromString(userAgentTest);
-
- String[] input = {
- "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
- "Mozilla/5.0 (compatible; Konqueror/3.5; Linux; X11; de) KHTML/3.5.2 (like Gecko) Kubuntu 6.06 Dapper",
- "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.2a1pre) Gecko/20110331 Firefox/4.2a1pre Fennec/4.1a1pre",
- "Opera/9.00 (X11; Linux i686; U; en)",
- "Wget/1.10.2",
- "Opera/9.80 (Android; Linux; Opera Mobi/ADR-1012221546; U; pl) Presto/2.7.60 Version/10.5",
- "Mozilla/5.0 (Linux; U; Android 2.2; en-us; DROID2 Build/VZW) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1"
- };
-
- String[] output = {
- "(mobile)",
- "(desktop)",
- "(mobile)",
- "(desktop)",
- "(desktop)",
- "(mobile)",
- "(mobile)",
- };
-
- test.assertOutput("data",input,"data_out",output);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/util/AliasEvalFuncTest.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/util/AliasEvalFuncTest.java b/test/pig/datafu/test/pig/util/AliasEvalFuncTest.java
deleted file mode 100644
index 3b23d35..0000000
--- a/test/pig/datafu/test/pig/util/AliasEvalFuncTest.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.util;
-
-import static org.testng.Assert.*;
-
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.io.IOException;
-
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.pigunit.PigTest;
-import org.apache.pig.data.DataType;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
-
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import datafu.test.pig.PigTests;
-import datafu.pig.util.AliasableEvalFunc;
-
-public class AliasEvalFuncTest extends PigTests
-{
- static class ReportBuilder extends AliasableEvalFunc<DataBag> {
- static final String ORDERED_ROUTES = "orderedRoutes";
-
- public DataBag exec(Tuple input) throws IOException {
- DataBag inputBag = getBag(input, ORDERED_ROUTES);
- DataBag outputBag = BagFactory.getInstance().newDefaultBag();
- for(Iterator<Tuple> tupleIter = inputBag.iterator(); tupleIter.hasNext(); ) {
- outputBag.add(tupleIter.next());
- }
- return outputBag;
- }
-
- public Schema getOutputSchema(Schema input) {
- try {
- Schema bagSchema = input.getField(0).schema;
- Schema outputSchema = new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
- .getName()
- .toLowerCase(), input),
- bagSchema,
- DataType.BAG));
- return outputSchema;
- } catch (Exception ex) {
- return null;
- }
- }
- }
-
- @Test
- public void getBagTest() throws Exception
- {
- ReportBuilder udf = new ReportBuilder();
- udf.setUDFContextSignature("test");
- List<Schema.FieldSchema> fieldSchemaList = new ArrayList<Schema.FieldSchema>();
- fieldSchemaList.add(new Schema.FieldSchema("msisdn", DataType.LONG));
- fieldSchemaList.add(new Schema.FieldSchema("ts", DataType.INTEGER));
- fieldSchemaList.add(new Schema.FieldSchema("center_lon", DataType.DOUBLE));
- fieldSchemaList.add(new Schema.FieldSchema("center_lat", DataType.DOUBLE));
- Schema schemaTuple = new Schema(fieldSchemaList);
- Schema schemaBag = new Schema(new Schema.FieldSchema(ReportBuilder.ORDERED_ROUTES, schemaTuple, DataType.BAG));
- udf.outputSchema(schemaBag);
-
- Tuple inputTuple = TupleFactory.getInstance().newTuple();
- DataBag inputBag = BagFactory.getInstance().newDefaultBag();
- inputBag.add(TupleFactory.getInstance().newTuple(Arrays.asList(71230000000L, 1382351612, 10.697, 20.713)));
- inputTuple.append(inputBag);
- DataBag outputBag = udf.exec(inputTuple);
- Assert.assertEquals(inputBag, outputBag);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/util/AssertTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/util/AssertTests.java b/test/pig/datafu/test/pig/util/AssertTests.java
deleted file mode 100644
index bf558a7..0000000
--- a/test/pig/datafu/test/pig/util/AssertTests.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.util;
-
-import static org.testng.Assert.*;
-
-import java.util.List;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import datafu.test.pig.PigTests;
-
-public class AssertTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- define ASRT datafu.pig.util.AssertUDF();
-
- data = LOAD 'input' AS (val:INT);
-
- data2 = FILTER data BY ASRT(val,'assertion appears to have failed, doh!');
-
- STORE data2 INTO 'output';
- */
- @Multiline private static String assertWithMessage;
-
- @Test
- public void shouldAssertWithMessageOnZero() throws Exception
- {
- try
- {
- PigTest test = createPigTestFromString(assertWithMessage);
-
- this.writeLinesToFile("input", "0");
-
- test.runScript();
-
- this.getLinesForAlias(test, "data2");
-
- fail("test should have failed, but it didn't");
- }
- catch (Exception e)
- {
- }
- }
-
- @Test
- public void shouldNotAssertWithMessageOnOne() throws Exception
- {
- PigTest test = createPigTestFromString(assertWithMessage);
-
- this.writeLinesToFile("input", "1");
-
- test.runScript();
-
- List<Tuple> result = this.getLinesForAlias(test, "data2");
- Assert.assertEquals(result.size(), 1);
- Assert.assertEquals(result.get(0).size(), 1);
- Assert.assertEquals(result.get(0).get(0), 1);
- }
-
- /**
- register $JAR_PATH
-
- define ASRT datafu.pig.util.AssertUDF();
-
- data = LOAD 'input' AS (val:INT);
-
- data2 = FILTER data BY ASRT(val);
-
- STORE data2 INTO 'output';
- */
- @Multiline private static String assertWithoutMessage;
-
- @Test
- public void shouldAssertWithoutMessageOnZero() throws Exception
- {
- try
- {
- PigTest test = createPigTestFromString(assertWithoutMessage);
-
- this.writeLinesToFile("input", "0");
-
- test.runScript();
-
- this.getLinesForAlias(test, "data2");
-
- fail("test should have failed, but it didn't");
- }
- catch (Exception e)
- {
- }
- }
-
- @Test
- public void shouldNotAssertWithoutMessageOnOne() throws Exception
- {
- PigTest test = createPigTestFromString(assertWithoutMessage);
-
- this.writeLinesToFile("input", "1");
-
- test.runScript();
-
- List<Tuple> result = this.getLinesForAlias(test, "data2");
- Assert.assertEquals(result.size(), 1);
- Assert.assertEquals(result.get(0).size(), 1);
- Assert.assertEquals(result.get(0).get(0), 1);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/util/CoalesceTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/util/CoalesceTests.java b/test/pig/datafu/test/pig/util/CoalesceTests.java
deleted file mode 100644
index 0dc0f4b..0000000
--- a/test/pig/datafu/test/pig/util/CoalesceTests.java
+++ /dev/null
@@ -1,532 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.util;
-
-import java.util.List;
-
-import junit.framework.Assert;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.pigunit.PigTest;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.testng.annotations.Test;
-
-import datafu.test.pig.PigTests;
-
-public class CoalesceTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- define COALESCE datafu.pig.util.Coalesce();
-
- data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:INT,val2:INT,val3:INT);
-
- data2 = FOREACH data GENERATE testcase, COALESCE(val1,val2,val3) as result;
-
- describe data2;
-
- data3 = FOREACH data2 GENERATE testcase, result;
-
- STORE data3 INTO 'output';
- */
- @Multiline private static String coalesceIntTest;
-
- @Test
- public void coalesceIntTest() throws Exception
- {
- PigTest test = createPigTestFromString(coalesceIntTest);
-
- this.writeLinesToFile("input", "1,1,2,3",
- "2,,2,3",
- "3,,,3",
- "4,,,",
- "5,1,,3",
- "6,1,,");
-
- test.runScript();
-
- List<Tuple> lines = this.getLinesForAlias(test, "data3");
-
- Assert.assertEquals(6, lines.size());
- for (Tuple t : lines)
- {
- switch((Integer)t.get(0))
- {
- case 1:
- Assert.assertEquals(1, t.get(1)); break;
- case 2:
- Assert.assertEquals(2, t.get(1)); break;
- case 3:
- Assert.assertEquals(3, t.get(1)); break;
- case 4:
- Assert.assertEquals(null, t.get(1)); break;
- case 5:
- Assert.assertEquals(1, t.get(1)); break;
- case 6:
- Assert.assertEquals(1, t.get(1)); break;
- default:
- Assert.fail("Did not expect: " + t.get(1));
- }
- }
- }
-
- /**
- register $JAR_PATH
-
- define COALESCE datafu.pig.util.Coalesce();
-
- data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:LONG);
-
- data2 = FOREACH data GENERATE testcase, COALESCE(val1,100L) as result;
-
- describe data2;
-
- data3 = FOREACH data2 GENERATE testcase, result;
-
- data4 = FOREACH data3 GENERATE testcase, result*100 as result;
-
- STORE data4 INTO 'output';
- */
- @Multiline private static String coalesceLongTest;
-
- @Test
- public void coalesceLongTest() throws Exception
- {
- PigTest test = createPigTestFromString(coalesceLongTest);
-
- this.writeLinesToFile("input", "1,5",
- "2,");
-
- test.runScript();
-
- List<Tuple> lines = this.getLinesForAlias(test, "data4");
-
- Assert.assertEquals(2, lines.size());
- for (Tuple t : lines)
- {
- switch((Integer)t.get(0))
- {
- case 1:
- Assert.assertEquals(500L, t.get(1)); break;
- case 2:
- Assert.assertEquals(10000L, t.get(1)); break;
- default:
- Assert.fail("Did not expect: " + t.get(1));
- }
- }
- }
-
- /**
- register $JAR_PATH
-
- define COALESCE datafu.pig.util.Coalesce();
-
- data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:LONG);
-
- data2 = FOREACH data GENERATE testcase, COALESCE(val1,100) as result;
-
- describe data2;
-
- data3 = FOREACH data2 GENERATE testcase, result;
-
- data4 = FOREACH data3 GENERATE testcase, result*100 as result;
-
- STORE data4 INTO 'output';
- */
- @Multiline private static String coalesceCastIntToLongTestFails;
-
- // The first parameter is a long and the fixed value is an int.
- // They cannot be merged without the lazy option.
- @Test(expectedExceptions=FrontendException.class)
- public void coalesceCastIntToLongTestFails() throws Exception
- {
- PigTest test = createPigTestFromString(coalesceCastIntToLongTestFails);
-
- this.writeLinesToFile("input", "1,5",
- "2,");
-
- test.runScript();
-
- List<Tuple> lines = this.getLinesForAlias(test, "data4");
-
- Assert.assertEquals(2, lines.size());
- for (Tuple t : lines)
- {
- switch((Integer)t.get(0))
- {
- case 1:
- Assert.assertEquals(500L, t.get(1)); break;
- case 2:
- Assert.assertEquals(10000L, t.get(1)); break;
- default:
- Assert.fail("Did not expect: " + t.get(1));
- }
- }
- }
-
- /**
- register $JAR_PATH
-
- define COALESCE datafu.pig.util.Coalesce('lazy');
-
- data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:LONG);
-
- data2 = FOREACH data GENERATE testcase, COALESCE(val1,100) as result;
-
- describe data2;
-
- data3 = FOREACH data2 GENERATE testcase, result;
-
- data4 = FOREACH data3 GENERATE testcase, result*100 as result;
-
- STORE data4 INTO 'output';
- */
- @Multiline private static String coalesceIntAndLongTest;
-
- // The first parameter is a long and the fixed value is an int.
- // They are merged to a long.
- @Test
- public void coalesceCastIntToLongTest1() throws Exception
- {
- PigTest test = createPigTestFromString(coalesceIntAndLongTest);
-
- this.writeLinesToFile("input", "1,5",
- "2,");
-
- test.runScript();
-
- List<Tuple> lines = this.getLinesForAlias(test, "data4");
-
- Assert.assertEquals(2, lines.size());
- for (Tuple t : lines)
- {
- switch((Integer)t.get(0))
- {
- case 1:
- Assert.assertEquals(500L, t.get(1)); break;
- case 2:
- Assert.assertEquals(10000L, t.get(1)); break;
- default:
- Assert.fail("Did not expect: " + t.get(1));
- }
- }
- }
-
- /**
- register $JAR_PATH
-
- define COALESCE datafu.pig.util.Coalesce('lazy');
-
- data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:INT);
-
- data2 = FOREACH data GENERATE testcase, COALESCE(val1,100L) as result;
-
- describe data2;
-
- data3 = FOREACH data2 GENERATE testcase, result;
-
- data4 = FOREACH data3 GENERATE testcase, result*100 as result;
-
- STORE data4 INTO 'output';
- */
- @Multiline private static String coalesceIntAndLongTest2;
-
- // The first parameter is an int, but the fixed parameter is a long.
- // They are merged to a long.
- @Test
- public void coalesceCastIntToLongTest2() throws Exception
- {
- PigTest test = createPigTestFromString(coalesceIntAndLongTest2);
-
- this.writeLinesToFile("input", "1,5",
- "2,");
-
- test.runScript();
-
- List<Tuple> lines = this.getLinesForAlias(test, "data4");
-
- Assert.assertEquals(2, lines.size());
- for (Tuple t : lines)
- {
- switch((Integer)t.get(0))
- {
- case 1:
- Assert.assertEquals(500L, t.get(1)); break;
- case 2:
- Assert.assertEquals(10000L, t.get(1)); break;
- default:
- Assert.fail("Did not expect: " + t.get(1));
- }
- }
- }
-
- /**
- register $JAR_PATH
-
- define COALESCE datafu.pig.util.Coalesce('lazy');
-
- data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:INT);
-
- data2 = FOREACH data GENERATE testcase, COALESCE(val1,100.0) as result;
-
- describe data2;
-
- data3 = FOREACH data2 GENERATE testcase, result;
-
- data4 = FOREACH data3 GENERATE testcase, result*100 as result;
-
- STORE data4 INTO 'output';
- */
- @Multiline private static String coalesceIntAndDoubleTest;
-
- // The first parameter is an int, but the fixed parameter is a long.
- // They are merged to a long.
- @Test
- public void coalesceCastIntToDoubleTest() throws Exception
- {
- PigTest test = createPigTestFromString(coalesceIntAndDoubleTest);
-
- this.writeLinesToFile("input", "1,5",
- "2,");
-
- test.runScript();
-
- List<Tuple> lines = this.getLinesForAlias(test, "data4");
-
- Assert.assertEquals(2, lines.size());
- for (Tuple t : lines)
- {
- switch((Integer)t.get(0))
- {
- case 1:
- Assert.assertEquals(500.0, t.get(1)); break;
- case 2:
- Assert.assertEquals(10000.0, t.get(1)); break;
- default:
- Assert.fail("Did not expect: " + t.get(1));
- }
- }
- }
-
- /**
- register $JAR_PATH
-
- define COALESCE datafu.pig.util.Coalesce();
-
- data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:LONG);
-
- data = FOREACH data GENERATE testcase, (val1 IS NOT NULL ? ToDate(val1) : (datetime)null) as val1;
-
- data2 = FOREACH data GENERATE testcase, COALESCE(val1,ToDate('1970-01-01T00:00:00.000Z')) as result;
-
- --describe data2;
-
- data3 = FOREACH data2 GENERATE testcase, result;
-
- STORE data3 INTO 'output';
- */
- @Multiline private static String coalesceCastIntToDatetimeTest;
-
- @Test
- public void coalesceCastIntToDatetimeTest() throws Exception
- {
- PigTest test = createPigTestFromString(coalesceCastIntToDatetimeTest);
-
- this.writeLinesToFile("input", "1,1375826183000",
- "2,");
-
- test.runScript();
-
- List<Tuple> lines = this.getLinesForAlias(test, "data3");
-
- Assert.assertEquals(2, lines.size());
- for (Tuple t : lines)
- {
- Integer testcase = (Integer)t.get(0);
- Assert.assertNotNull(testcase);
- switch(testcase)
- {
- case 1:
- Assert.assertEquals("2013-08-06T21:56:23.000Z", ((DateTime)t.get(1)).toDateTime(DateTimeZone.UTC).toString()); break;
- case 2:
- Assert.assertEquals("1970-01-01T00:00:00.000Z", t.get(1).toString()); break;
- default:
- Assert.fail("Did not expect: " + t.get(1));
- }
- }
- }
-
- /**
- register $JAR_PATH
-
- define COALESCE datafu.pig.util.Coalesce('lazy');
-
- data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:LONG);
-
- data = FOREACH data GENERATE testcase, (val1 IS NOT NULL ? ToDate(val1) : (datetime)null) as val1;
-
- data2 = FOREACH data GENERATE testcase, COALESCE(val1,ToDate('1970-01-01T00:00:00.000Z')) as result;
-
- --describe data2;
-
- data3 = FOREACH data2 GENERATE testcase, result;
-
- STORE data3 INTO 'output';
- */
- @Multiline private static String coalesceCastIntToDatetimeLazyTest;
-
- @Test
- public void coalesceCastIntToDatetimeLazyTest() throws Exception
- {
- PigTest test = createPigTestFromString(coalesceCastIntToDatetimeLazyTest);
-
- this.writeLinesToFile("input", "1,1375826183000",
- "2,");
-
- test.runScript();
-
- List<Tuple> lines = this.getLinesForAlias(test, "data3");
-
- Assert.assertEquals(2, lines.size());
- for (Tuple t : lines)
- {
- Integer testcase = (Integer)t.get(0);
- Assert.assertNotNull(testcase);
- switch(testcase)
- {
- case 1:
- Assert.assertEquals("2013-08-06T21:56:23.000Z", ((DateTime)t.get(1)).toDateTime(DateTimeZone.UTC).toString()); break;
- case 2:
- Assert.assertEquals("1970-01-01T00:00:00.000Z", t.get(1).toString()); break;
- default:
- Assert.fail("Did not expect: " + t.get(1));
- }
- }
- }
-
- /**
- register $JAR_PATH
-
- define COALESCE datafu.pig.util.Coalesce();
-
- data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:INT,val2:LONG);
-
- data2 = FOREACH data GENERATE testcase, COALESCE(val1,val2) as result;
-
- describe data2;
-
- data3 = FOREACH data2 GENERATE testcase, result;
-
- STORE data3 INTO 'output';
- */
- @Multiline private static String coalesceBagIncompatibleTypeTest;
-
- @Test(expectedExceptions=FrontendException.class)
- public void coalesceBagIncompatibleTypeTest() throws Exception
- {
- PigTest test = createPigTestFromString(coalesceBagIncompatibleTypeTest);
-
- this.writeLinesToFile("input", "1,1,2L}");
-
- test.runScript();
-
- this.getLinesForAlias(test, "data3");
- }
-
- /**
- register $JAR_PATH
-
- define COALESCE datafu.pig.util.Coalesce('lazy');
- define EmptyBagToNullFields datafu.pig.bags.EmptyBagToNullFields();
-
- input1 = LOAD 'input1' using PigStorage(',') AS (val1:INT,val2:INT);
- input2 = LOAD 'input2' using PigStorage(',') AS (val1:INT,val2:INT);
- input3 = LOAD 'input3' using PigStorage(',') AS (val1:INT,val2:INT);
-
- data4 = COGROUP input1 BY val1,
- input2 BY val1,
- input3 BY val1;
-
- dump data4;
-
- data4 = FOREACH data4 GENERATE
- FLATTEN(input1),
- FLATTEN(EmptyBagToNullFields(input2)),
- FLATTEN(EmptyBagToNullFields(input3));
-
- dump data4;
-
- describe data4;
-
- data5 = FOREACH data4 GENERATE input1::val1 as val1, COALESCE(input2::val2,0L) as val2, COALESCE(input3::val2,0L) as val3;
-
- --describe data5;
-
- STORE data5 INTO 'output';
- */
- @Multiline private static String leftJoinTest;
-
- @Test
- public void leftJoinTest() throws Exception
- {
- PigTest test = createPigTestFromString(leftJoinTest);
-
- this.writeLinesToFile("input1", "1,1",
- "2,2",
- "5,5");
-
- this.writeLinesToFile("input2", "1,10",
- "3,30",
- "5,50");
-
- this.writeLinesToFile("input3", "2,100",
- "5,500");
-
- test.runScript();
-
- List<Tuple> lines = this.getLinesForAlias(test, "data5");
-
- Assert.assertEquals(3, lines.size());
- for (Tuple t : lines)
- {
- switch((Integer)t.get(0))
- {
- case 1:
- Assert.assertEquals(10L, t.get(1));
- Assert.assertEquals(0L, t.get(2));
- break;
- case 2:
- Assert.assertEquals(0L, t.get(1));
- Assert.assertEquals(100L, t.get(2));
- break;
- case 5:
- Assert.assertEquals(50L, t.get(1));
- Assert.assertEquals(500L, t.get(2));
- break;
- default:
- Assert.fail("Did not expect: " + t.get(0));
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/util/InTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/util/InTests.java b/test/pig/datafu/test/pig/util/InTests.java
deleted file mode 100644
index a21a0b2..0000000
--- a/test/pig/datafu/test/pig/util/InTests.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.util;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.test.pig.PigTests;
-
-public class InTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- define I datafu.pig.util.InUDF();
-
- data = LOAD 'input' AS (B: bag {T: tuple(v:INT)});
-
- data2 = FOREACH data {
- C = FILTER B By I(v, 1,2,3);
- GENERATE C;
- }
-
- describe data2;
-
- STORE data2 INTO 'output';
- */
- @Multiline private static String inIntTest;
-
- @Test
- public void inIntTest() throws Exception
- {
- PigTest test = createPigTestFromString(inIntTest);
-
- writeLinesToFile("input",
- "({(1),(2),(3),(4),(5)})",
- "({(1),(2)})",
- "({(4),(5)})");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "({(1),(2),(3)})",
- "({(1),(2)})",
- "({})");
- }
-
- /**
- register $JAR_PATH
-
- define I datafu.pig.util.InUDF();
-
- data = LOAD 'input' AS (B: bag {T: tuple(v:chararray)});
-
- data2 = FOREACH data {
- C = FILTER B By I(v, 'will','matt','sam');
- GENERATE C;
- }
-
- describe data2;
-
- STORE data2 INTO 'output';
- */
- @Multiline private static String inStringTest;
-
- @Test
- public void inStringTest() throws Exception
- {
- PigTest test = createPigTestFromString(inStringTest);
-
- writeLinesToFile("input",
- "({(alice),(bob),(will),(matt),(sam)})",
- "({(will),(matt)})",
- "({(alice),(bob)})");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "({(will),(matt),(sam)})",
- "({(will),(matt)})",
- "({})");
- }
-
- /**
- register $JAR_PATH
-
- define I datafu.pig.util.InUDF();
-
- data = LOAD 'input' AS (owner:chararray, color:chararray);
- describe data;
- data2 = FILTER data BY I(color, 'red','blue');
- describe data2;
- STORE data2 INTO 'output';
- */
- @Multiline private static String inTopLevelTest;
-
- @Test
- public void inTopLevelTest() throws Exception
- {
- PigTest test = createPigTestFromString(inTopLevelTest);
-
- writeLinesToFile("input",
- "alice\tred",
- "bob\tblue",
- "charlie\tgreen",
- "dave\tred");
- test.runScript();
-
- assertOutput(test, "data2",
- "(alice,red)",
- "(bob,blue)",
- "(dave,red)");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/util/IntBoolConversionPigTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/util/IntBoolConversionPigTests.java b/test/pig/datafu/test/pig/util/IntBoolConversionPigTests.java
deleted file mode 100644
index 4017867..0000000
--- a/test/pig/datafu/test/pig/util/IntBoolConversionPigTests.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.util;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.test.pig.PigTests;
-
-public class IntBoolConversionPigTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- define IntToBool datafu.pig.util.IntToBool();
-
- data = LOAD 'input' AS (val:INT);
-
- data2 = FOREACH data GENERATE IntToBool(val);
-
- STORE data2 INTO 'output';
- */
- @Multiline private static String intToBoolTest;
-
- @Test
- public void intToBoolTest() throws Exception
- {
- PigTest test = createPigTestFromString(intToBoolTest);
-
- String[] input = {
- "", // null
- "0",
- "1"
- };
-
- String[] output = {
- "(false)",
- "(false)",
- "(true)"
- };
-
- test.assertOutput("data",input,"data2",output);
- }
-
- /**
- register $JAR_PATH
-
- define IntToBool datafu.pig.util.IntToBool();
- define BoolToInt datafu.pig.util.BoolToInt();
-
- data = LOAD 'input' AS (val:INT);
-
- data2 = FOREACH data GENERATE IntToBool(val) as val;
- data3 = FOREACH data2 GENERATE BoolToInt(val) as val;
-
- STORE data3 INTO 'output';
- */
- @Multiline private static String intToBoolToIntTest;
-
- @Test
- public void intToBoolToIntTest() throws Exception
- {
- PigTest test = createPigTestFromString(intToBoolToIntTest);
-
- String[] input = {
- "", // null
- "0",
- "1",
- "2",
- "-1",
- "-2",
- "0",
- ""
- };
-
- String[] output = {
- "(0)",
- "(0)",
- "(1)",
- "(1)",
- "(1)",
- "(1)",
- "(0)",
- "(0)"
- };
-
- test.assertOutput("data",input,"data3",output);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/util/TransposeTest.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/util/TransposeTest.java b/test/pig/datafu/test/pig/util/TransposeTest.java
deleted file mode 100644
index f131cd0..0000000
--- a/test/pig/datafu/test/pig/util/TransposeTest.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.util;
-
-import java.util.List;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import datafu.test.pig.PigTests;
-
-public class TransposeTest extends PigTests
-{
- /**
- register $JAR_PATH
-
- define Transpose datafu.pig.util.TransposeTupleToBag();
-
- data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:INT,val2:INT,val3:INT);
-
- data2 = FOREACH data GENERATE testcase, Transpose(val1 .. val3) as transposed;
-
- describe data2;
-
- data3 = FOREACH data2 GENERATE testcase, transposed;
-
- STORE data3 INTO 'output';
- */
- @Multiline private static String transposeTest;
-
- @Test
- public void transposeTest() throws Exception
- {
- PigTest test = createPigTestFromString(transposeTest);
- writeLinesToFile("input", "1,10,11,12",
- "2,20,21,22");
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data3");
- for (Tuple tuple : output) {
- int testCase = (Integer)tuple.get(0);
- DataBag bag = (DataBag)tuple.get(1);
- Assert.assertEquals(bag.size(), 3);
- int i=0;
- for (Tuple t : bag) {
- String expectedKey = String.format("val%d",i+1);
- Assert.assertEquals((String)t.get(0), expectedKey);
- int actualValue = (Integer)t.get(1);
- Assert.assertEquals(actualValue, testCase*10+i);
- i++;
- }
- }
- }
-
- /**
- register $JAR_PATH
-
- define Transpose datafu.pig.util.TransposeTupleToBag();
-
- data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:INT,val2:INT,val3:DOUBLE);
-
- data2 = FOREACH data GENERATE testcase, Transpose(val1 .. val3) as transposed;
-
- describe data2;
-
- data3 = FOREACH data2 GENERATE testcase, transposed;
-
- STORE data3 INTO 'output';
- */
- @Multiline private static String transposeBadTypeTest;
-
- @Test(expectedExceptions={org.apache.pig.impl.logicalLayer.FrontendException.class})
- public void transposeBadTypeTest() throws Exception
- {
- PigTest test = createPigTestFromString(transposeBadTypeTest);
- writeLinesToFile("input", "1,10,11,12.0",
- "2,20,21,22.0");
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data3");
- for (Tuple tuple : output) {
- int testCase = (Integer)tuple.get(0);
- DataBag bag = (DataBag)tuple.get(1);
- Assert.assertEquals(bag.size(), 3);
- int i=0;
- for (Tuple t : bag) {
- String expectedKey = String.format("val%d",i+1);
- Assert.assertEquals((String)t.get(0), expectedKey);
- int actualValue = (Integer)t.get(1);
- Assert.assertEquals(actualValue, testCase*10+i);
- i++;
- }
- }
- }
-}
[15/19] DATAFU-27 Migrate build system to Gradle
Posted by wv...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/StreamingQuantile.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/StreamingQuantile.java b/datafu-pig/src/main/java/datafu/pig/stats/StreamingQuantile.java
new file mode 100644
index 0000000..e4a65b4
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/StreamingQuantile.java
@@ -0,0 +1,418 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.pig.AccumulatorEvalFunc;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
+
+/**
+ * Computes approximate {@link <a href="http://en.wikipedia.org/wiki/Quantile" target="_blank">quantiles</a>}
+ * for a (not necessarily sorted) input bag, using the Munro-Paterson algorithm.
+ *
+ * <p>
+ * The algorithm is described here:
+ * {@link <a href="http://www.cs.ucsb.edu/~suri/cs290/MunroPat.pdf" target="_blank">http://www.cs.ucsb.edu/~suri/cs290/MunroPat.pdf</a>}
+ * </p>
+ *
+ * <p>
+ * The implementation is based on the one in Sawzall, available here:
+ * {@link <a href="http://szl.googlecode.com/svn-history/r41/trunk/src/emitters/szlquantile.cc">szlquantile.cc</a>}
+ * </p>
+ *
+ * <p>
+ * N.B., all the data is pushed to a single reducer per key, so make sure some partitioning is
+ * done (e.g., group by 'day') if the data is too large. That is, this isn't distributed quantiles.
+ * </p>
+ *
+ * <p>
+ * Note that unlike datafu's standard Quantile algorithm, the Munro-Paterson algorithm gives
+ * <b>approximate</b> quantiles and does not require the input bag to be sorted. Because it implements
+ * accumulate, StreamingQuantile can be much more efficient than Quantile for large amounts of data which
+ * do not fit in memory. Quantile must spill to disk when the input data is too large to fit in memory,
+ * which will contribute to longer runtimes.
+ * </p>
+ *
+ * <p>The constructor takes a single integer argument that specifies the number of evenly-spaced
+ * quantiles to compute, e.g.,</p>
+ *
+ * <ul>
+ * <li>StreamingQuantile('3') yields the min, the median, and the max
+ * <li>StreamingQuantile('5') yields the min, the 25th, 50th, 75th percentiles, and the max
+ * <li>StreamingQuantile('101') yields the min, the max, and all 99 percentiles.
+ * </ul>
+ *
+ * <p>Alternatively the constructor can take the explicit list of quantiles to compute, e.g.</p>
+ *
+ * <ul>
+ * <li>StreamingQuantile('0.0','0.5','1.0') yields the min, the median, and the max
+ * <li>StreamingQuantile('0.0','0.25','0.5','0.75','1.0') yields the min, the 25th, 50th, 75th percentiles, and the max
+ * </ul>
+ *
+ * <p>The list of quantiles need not span the entire range from 0.0 to 1.0, nor do they need to be evenly spaced, e.g.</p>
+ *
+ * <ul>
+ * <li>StreamingQuantile('0.5','0.90','0.95','0.99') yields the median, the 90th, 95th, and the 99th percentiles
+ * <li>StreamingQuantile('0.0013','0.0228','0.1587','0.5','0.8413','0.9772','0.9987') yields the 0.13th, 2.28th, 15.87th, 50th, 84.13th, 97.72nd, and 99.87th percentiles
+ * </ul>
+ *
+ * <p>Be aware when specifying the list of quantiles in this way that more quantiles may be computed internally than are actually returned.
+ * The GCD of the quantiles is found and this determines the number of evenly spaced quantiles to compute. The requested quantiles
+ * are then returned from this set. For instance:</p>
+ *
+ * <ul>
+ * <li>If the quantiles 0.2 and 0.6 are requested then the quantiles 0.0, 0.2, 0.4, 0.6, 0.8, and 1.0 are computed
+ * because 0.2 is the GCD of 0.2, 0.6, and 1.0.</li>
+ * <li>If 0.2 and 0.7 are requested then the quantiles 0.0, 0.1, 0.2, ... , 0.9, 1.0 are computed because 0.1 is the
+ * GCD of 0.2, 0.7, and 1.0.</li>
+ * <li>If 0.999 is requested the quantiles 0.0, 0.001, 0.002, ... , 0.998, 0.999, 1.0 are computed because 0.001 is
+ * the GCD of 0.999 and 1.0.</li>
+ * </p>
+ * </ul>
+ *
+ * <p>The error on the approximation goes down as the number of buckets computed goes up.</p>
+ *
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ *
+ * define Quantile datafu.pig.stats.StreamingQuantile('5');
+
+ * -- input: 9,10,2,3,5,8,1,4,6,7
+ * input = LOAD 'input' AS (val:int);
+ *
+ * grouped = GROUP input ALL;
+ *
+ * -- produces: (1.0,3.0,5.0,8.0,10.0)
+ * quantiles = FOREACH grouped generate Quantile(input);
+ * }
+ * </pre></p>
+ *
+ * @see StreamingMedian
+ * @see Quantile
+ */
+public class StreamingQuantile extends AccumulatorEvalFunc<Tuple>
+{
+ private final int numQuantiles;
+ private final QuantileEstimator estimator;
+ private List<Double> quantiles;
+
+ // For the output schema, label the quantiles 0, 1, 2, ... n
+ // Otherwise label the quantiles based on the quantile value.
+ // e.g. 50% quantile 0.5 will be labeled as 0_5
+ private boolean ordinalOutputSchema;
+
+ public StreamingQuantile(String... k)
+ {
+ if (k.length == 1 && Double.parseDouble(k[0]) > 1.0)
+ {
+ this.ordinalOutputSchema = true;
+ this.numQuantiles = Integer.parseInt(k[0]);
+ }
+ else
+ {
+ this.quantiles = QuantileUtil.getQuantilesFromParams(k);
+ this.numQuantiles = getNumQuantiles(this.quantiles);
+ }
+ this.estimator = new QuantileEstimator(this.numQuantiles);
+ }
+
+ private static int getNumQuantiles(List<Double> quantiles)
+ {
+ quantiles = new ArrayList<Double>(quantiles);
+ Collections.sort(quantiles);
+ int start = 0;
+ int end = quantiles.size()-1;
+ while (quantiles.get(start) == 0.0) start++;
+ while (quantiles.get(end) == 1.0) end--;
+ double gcd = 1.0;
+ for (int i=end; i>=start; i--)
+ {
+ gcd = gcd(gcd,quantiles.get(i));
+ }
+ int numQuantiles = (int)(1/gcd) + 1;
+ return numQuantiles;
+ }
+
+ private static double gcd(double a, double b)
+ {
+ if (round(a) == 0.0)
+ {
+ throw new IllegalArgumentException("Quantiles are smaller than the allowed precision");
+ }
+ if (round(b) == 0.0)
+ {
+ throw new IllegalArgumentException("Quantiles are smaller than the allowed precision");
+ }
+ while (round(b) != 0.0)
+ {
+ double t = b;
+ b = a % b;
+ a = t;
+ }
+ return round(a);
+ }
+
+ private static double round(double d)
+ {
+ return Math.round(d*100000.0)/100000.0;
+ }
+
+ @Override
+ public void accumulate(Tuple b) throws IOException
+ {
+ DataBag bag = (DataBag) b.get(0);
+ if (bag == null || bag.size() == 0)
+ return;
+
+ for (Tuple t : bag) {
+ Object o = t.get(0);
+ if (!(o instanceof Number)) {
+ throw new IllegalStateException("bag must have numerical values (and be non-null)");
+ }
+ estimator.add(((Number) o).doubleValue());
+ }
+ }
+
+ @Override
+ public void cleanup()
+ {
+ estimator.clear();
+ }
+
+ @Override
+ public Tuple getValue()
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(this.quantiles != null ? this.quantiles.size() : this.numQuantiles);
+ try {
+ if (this.quantiles == null)
+ {
+ int j = 0;
+ for (double quantileValue : estimator.getQuantiles())
+ {
+ t.set(j, quantileValue);
+ j++;
+ }
+ }
+ else
+ {
+ HashMap<Double,Double> quantileValues = new HashMap<Double,Double>(this.quantiles.size());
+ double quantileKey = 0.0;
+ for (double quantileValue : estimator.getQuantiles()) {
+ quantileValues.put(round(quantileKey), quantileValue);
+ quantileKey += 1.0/(this.numQuantiles-1);
+ }
+ int j = 0;
+ for (double d : this.quantiles)
+ {
+ Double quantileValue = quantileValues.get(round(d));
+ t.set(j, quantileValue);
+ j++;
+ }
+ }
+ } catch (IOException e) {
+ return null;
+ }
+ return t;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ Schema tupleSchema = new Schema();
+ if (ordinalOutputSchema)
+ {
+ for (int i = 0; i < this.numQuantiles; i++)
+ {
+ tupleSchema.add(new Schema.FieldSchema("quantile_" + i, DataType.DOUBLE));
+ }
+ }
+ else
+ {
+ for (Double x : this.quantiles)
+ tupleSchema.add(new Schema.FieldSchema("quantile_" + x.toString().replace(".", "_"), DataType.DOUBLE));
+ }
+
+ try {
+ return new Schema(new FieldSchema(null, tupleSchema, DataType.TUPLE));
+ } catch(FrontendException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ static class QuantileEstimator
+ {
+ private static final long MAX_TOT_ELEMS = 1024L * 1024L * 1024L * 1024L;
+
+ private final List<List<Double>> buffer = new ArrayList<List<Double>>();
+ private final int numQuantiles;
+ private final int maxElementsPerBuffer;
+ private int totalElements;
+ private double min;
+ private double max;
+
+ public QuantileEstimator(int numQuantiles)
+ {
+ this.numQuantiles = numQuantiles;
+ this.maxElementsPerBuffer = computeMaxElementsPerBuffer();
+ }
+
+ private int computeMaxElementsPerBuffer()
+ {
+ double epsilon = 1.0 / (numQuantiles - 1.0);
+ int b = 2;
+ while ((b - 2) * (0x1L << (b - 2)) + 0.5 <= epsilon * MAX_TOT_ELEMS) {
+ ++b;
+ }
+ return (int) (MAX_TOT_ELEMS / (0x1L << (b - 1)));
+ }
+
+ private void ensureBuffer(int level)
+ {
+ while (buffer.size() < level + 1) {
+ buffer.add(null);
+ }
+ if (buffer.get(level) == null) {
+ buffer.set(level, new ArrayList<Double>());
+ }
+ }
+
+ private void collapse(List<Double> a, List<Double> b, List<Double> out)
+ {
+ int indexA = 0, indexB = 0, count = 0;
+ Double smaller = null;
+ while (indexA < maxElementsPerBuffer || indexB < maxElementsPerBuffer) {
+ if (indexA >= maxElementsPerBuffer ||
+ (indexB < maxElementsPerBuffer && a.get(indexA) >= b.get(indexB))) {
+ smaller = b.get(indexB++);
+ } else {
+ smaller = a.get(indexA++);
+ }
+
+ if (count++ % 2 == 0) {
+ out.add(smaller);
+ }
+ }
+ a.clear();
+ b.clear();
+ }
+
+ private void recursiveCollapse(List<Double> buf, int level)
+ {
+ ensureBuffer(level + 1);
+
+ List<Double> merged;
+ if (buffer.get(level + 1).isEmpty()) {
+ merged = buffer.get(level + 1);
+ } else {
+ merged = new ArrayList<Double>(maxElementsPerBuffer);
+ }
+
+ collapse(buffer.get(level), buf, merged);
+ if (buffer.get(level + 1) != merged) {
+ recursiveCollapse(merged, level + 1);
+ }
+ }
+
+ public void add(double elem)
+ {
+ if (totalElements == 0 || elem < min) {
+ min = elem;
+ }
+ if (totalElements == 0 || max < elem) {
+ max = elem;
+ }
+
+ if (totalElements > 0 && totalElements % (2 * maxElementsPerBuffer) == 0) {
+ Collections.sort(buffer.get(0));
+ Collections.sort(buffer.get(1));
+ recursiveCollapse(buffer.get(0), 1);
+ }
+
+ ensureBuffer(0);
+ ensureBuffer(1);
+ int index = buffer.get(0).size() < maxElementsPerBuffer ? 0 : 1;
+ buffer.get(index).add(elem);
+ totalElements++;
+ }
+
+ public void clear()
+ {
+ buffer.clear();
+ totalElements = 0;
+ }
+
+ public List<Double> getQuantiles()
+ {
+ List<Double> quantiles = new ArrayList<Double>();
+ quantiles.add(min);
+
+ if (buffer.get(0) != null) {
+ Collections.sort(buffer.get(0));
+ }
+ if (buffer.get(1) != null) {
+ Collections.sort(buffer.get(1));
+ }
+
+ int[] index = new int[buffer.size()];
+ long S = 0;
+ for (int i = 1; i <= numQuantiles - 2; i++) {
+ long targetS = (long) Math.ceil(i * (totalElements / (numQuantiles - 1.0)));
+
+ while (true) {
+ double smallest = max;
+ int minBufferId = -1;
+ for (int j = 0; j < buffer.size(); j++) {
+ if (buffer.get(j) != null && index[j] < buffer.get(j).size()) {
+ if (!(smallest < buffer.get(j).get(index[j]))) {
+ smallest = buffer.get(j).get(index[j]);
+ minBufferId = j;
+ }
+ }
+ }
+
+ long incrementS = minBufferId <= 1 ? 1L : (0x1L << (minBufferId - 1));
+ if (S + incrementS >= targetS) {
+ quantiles.add(smallest);
+ break;
+ } else {
+ index[minBufferId]++;
+ S += incrementS;
+ }
+ }
+ }
+
+ quantiles.add(max);
+ return quantiles;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/VAR.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/VAR.java b/datafu-pig/src/main/java/datafu/pig/stats/VAR.java
new file mode 100644
index 0000000..6f22f25
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/VAR.java
@@ -0,0 +1,404 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.pig.Accumulator;
+import org.apache.pig.Algebraic;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.FuncSpec;
+import org.apache.pig.PigException;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.backend.executionengine.ExecException;
+
+
+/**
+* Generates the {@link <a href="http://en.wikipedia.org/wiki/Variance" target="_blank">Variance</a>}
+* of a set of Values. This UDF uses the fact that variance(x) = average(x^2) - average(x)^2
+* This class implements * {@link org.apache.pig.Algebraic}, so if possible the execution will performed in a distributed fashion.
+* VAR implements the {@link org.apache.pig.Accumulator} interface as well.
+*
+* Input: Bag of int, long, double, float or bytearray
+* Output: Double
+*
+* <p>
+* Example:
+* <pre>
+* define VAR datafu.pig.stats.VAR();
+*
+* -- input: 1,2,3,4,10,5,6,7,8,9
+* input = LOAD 'input' AS (val:int);
+* grouped = GROUP input ALL;
+* variance = FOREACH grouped GENERATE VAR(input.val) AS variance;
+* </pre>
+* </p>
+*/
+public class VAR extends EvalFunc<Double> implements Algebraic, Accumulator<Double> {
+ private static TupleFactory mTupleFactory = TupleFactory.getInstance();
+
+ @Override
+ public Double exec(Tuple input) throws IOException {
+ try {
+ Double sum = sum(input);
+ Double sumSquare = sumSquare(input);
+
+ if(sum == null) {
+ // either we were handed an empty bag or a bag
+ // filled with nulls - return null in this case
+ return null;
+ }
+ long count = count(input);
+
+ Double var = null;
+ if (count > 0){
+ Double avg = new Double(sum / count);
+ Double avgSquare = new Double(sumSquare / count);
+ var = avgSquare - avg*avg;
+ }
+
+ return var;
+ } catch (ExecException ee) {
+ throw ee;
+ }
+ }
+
+ public String getInitial() {
+ return Initial.class.getName();
+ }
+
+ public String getIntermed() {
+ return Intermediate.class.getName();
+ }
+
+ public String getFinal() {
+ return Final.class.getName();
+ }
+
+ static public class Initial extends EvalFunc<Tuple> {
+ @Override
+ public Tuple exec(Tuple input) throws IOException {
+ Tuple t = mTupleFactory.newTuple(3);
+ try {
+ // input is a bag with one tuple containing
+ // the column we are trying to get variance
+ DataBag bg = (DataBag) input.get(0);
+ DataByteArray dba = null;
+ Iterator<Tuple> iter = bg.iterator();
+ if(iter.hasNext()) {
+ Tuple tp = iter.next();
+ dba = (DataByteArray)tp.get(0);
+ }
+
+ if (iter.hasNext())
+ {
+ throw new RuntimeException("Expected only one tuple in bag");
+ }
+
+ Double d = dba !=null ? Double.valueOf(dba.toString()) : null;
+ if (dba == null){
+ t.set(0, null);
+ t.set(1, null);
+ t.set(2, 0L);
+ }
+ else {
+ t.set(0, d);
+ t.set(1, d*d);
+ t.set(2, 1L);
+ }
+ return t;
+ } catch(NumberFormatException nfe) {
+ nfe.printStackTrace();
+ // invalid input,
+ // treat this input as null
+ try {
+ t.set(0, null);
+ t.set(1, null);
+ t.set(2, 0L);
+ } catch (ExecException e) {
+ throw e;
+ }
+ return t;
+ } catch (ExecException ee) {
+ ee.printStackTrace();
+ throw ee;
+ } catch (Exception e) {
+ e.printStackTrace();
+ int errCode = 2106;
+ String msg = "Error while computing variance in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+ }
+
+ }
+ }
+
+ static public class Intermediate extends EvalFunc<Tuple> {
+ @Override
+ public Tuple exec(Tuple input) throws IOException {
+ try {
+ DataBag b = (DataBag)input.get(0);
+ return combine(b);
+ } catch (ExecException ee) {
+ throw ee;
+ } catch (Exception e) {
+ int errCode = 2106;
+ String msg = "Error while computing variacne in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+
+ }
+ }
+ }
+
+ static public class Final extends EvalFunc<Double> {
+ @Override
+ public Double exec(Tuple input) throws IOException {
+ try {
+ DataBag b = (DataBag)input.get(0);
+ Tuple combined = combine(b);
+
+ Double sum = (Double)combined.get(0);
+ Double sumSquare = (Double)combined.get(1);
+ if(sum == null) {
+ return null;
+ }
+ Long count = (Long)combined.get(2);
+
+ Double var = null;
+
+ if (count > 0) {
+ Double avg = new Double(sum / count);
+ Double avgSquare = new Double(sumSquare / count);
+ var = avgSquare - avg*avg;
+ }
+ return var;
+ } catch (ExecException ee) {
+ throw ee;
+ } catch (Exception e) {
+ int errCode = 2106;
+ String msg = "Error while computing variance in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+ }
+ }
+ }
+
+ static protected Tuple combine(DataBag values) throws ExecException{
+ double sum = 0;
+ double sumSquare = 0;
+ long totalCount = 0;
+
+ // combine is called from Intermediate and Final
+ // In either case, Initial would have been called
+ // before and would have sent in valid tuples
+ // Hence we don't need to check if incoming bag
+ // is empty
+
+ Tuple output = mTupleFactory.newTuple(3);
+ boolean sawNonNull = false;
+ for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
+ Tuple t = it.next();
+ Double d = (Double)t.get(0);
+ Double dSquare = (Double)t.get(1);
+ Long count = (Long)t.get(2);
+
+ // we count nulls in var as contributing 0
+ // a departure from SQL for performance of
+ // COUNT() which implemented by just inspecting
+ // size of the bag
+ if(d == null) {
+ d = 0.0;
+ } else {
+ sawNonNull = true;
+ }
+ sum += d;
+ sumSquare += dSquare;
+ totalCount += count;
+ }
+ if(sawNonNull) {
+ output.set(0, new Double(sum));
+ output.set(1, new Double(sumSquare));
+ } else {
+ output.set(0, null);
+ output.set(1, null);
+ }
+ output.set(2, Long.valueOf(totalCount));
+ return output;
+ }
+
+ static protected long count(Tuple input) throws ExecException {
+ DataBag values = (DataBag)input.get(0);
+ long cnt = 0;
+ Iterator<Tuple> it = values.iterator();
+ while (it.hasNext()){
+ Tuple t = (Tuple)it.next();
+ if (t != null && t.size() > 0 && t.get(0) != null)
+ cnt ++;
+ }
+
+ return cnt;
+ }
+
+ static protected Double sum(Tuple input) throws ExecException, IOException {
+ DataBag values = (DataBag)input.get(0);
+
+ // if we were handed an empty bag, return NULL
+ if(values.size() == 0) {
+ return null;
+ }
+
+ double sum = 0;
+ boolean sawNonNull = false;
+ for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
+ Tuple t = it.next();
+ try{
+ DataByteArray dba = (DataByteArray)t.get(0);
+ Double d = dba != null ? Double.valueOf(dba.toString()) : null;
+ if (d == null) continue;
+ sawNonNull = true;
+ sum += d;
+ }catch(RuntimeException exp) {
+ int errCode = 2103;
+ String msg = "Problem while computing sum of values.";
+ throw new ExecException(msg, errCode, PigException.BUG, exp);
+ }
+ }
+
+ if(sawNonNull) {
+ return new Double(sum);
+ } else {
+ return null;
+ }
+ }
+
+ static protected Double sumSquare(Tuple input) throws ExecException, IOException {
+ DataBag values = (DataBag)input.get(0);
+
+ // if we were handed an empty bag, return NULL
+ if(values.size() == 0) {
+ return null;
+ }
+
+ double sumSquare = 0;
+ boolean sawNonNull = false;
+ for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
+ Tuple t = it.next();
+ try{
+ DataByteArray dba = (DataByteArray)t.get(0);
+ Double d = dba != null ? Double.valueOf(dba.toString()) : null;
+ if (d == null) continue;
+ sawNonNull = true;
+ sumSquare += d*d;
+ }catch(RuntimeException exp) {
+ int errCode = 2103;
+ String msg = "Problem while computing sum of squared values.";
+ throw new ExecException(msg, errCode, PigException.BUG, exp);
+ }
+ }
+
+ if(sawNonNull) {
+ return new Double(sumSquare);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public Schema outputSchema(Schema input) {
+ return new Schema(new Schema.FieldSchema(null, DataType.DOUBLE));
+ }
+
+ @Override
+ public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
+ List<FuncSpec> funcList = new ArrayList<FuncSpec>();
+ funcList.add(new FuncSpec(this.getClass().getName(), Schema.generateNestedSchema(DataType.BAG, DataType.BYTEARRAY)));
+ funcList.add(new FuncSpec(DoubleVAR.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.DOUBLE)));
+ funcList.add(new FuncSpec(FloatVAR.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.FLOAT)));
+ funcList.add(new FuncSpec(IntVAR.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.INTEGER)));
+ funcList.add(new FuncSpec(LongVAR.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.LONG)));
+ return funcList;
+ }
+
+ /* Accumulator interface implementation */
+ private Double intermediateSumSquare = null;
+ private Double intermediateSum = null;
+ private Long intermediateCount = null;
+
+ @Override
+ public void accumulate(Tuple b) throws IOException {
+ try {
+ Double sum = sum(b);
+ if(sum == null) {
+ return;
+ }
+
+ Double sumSquare = sumSquare(b);
+ if(sumSquare == null) {
+ return;
+ }
+
+ // set default values
+ if (intermediateSum == null || intermediateCount == null) {
+ intermediateSumSquare = 0.0;
+ intermediateSum = 0.0;
+ intermediateCount = (long) 0;
+ }
+
+ long count = (Long)count(b);
+
+ if (count > 0) {
+ intermediateCount += count;
+ intermediateSum += sum;
+ intermediateSumSquare += sumSquare;
+ }
+ } catch (ExecException ee) {
+ throw ee;
+ } catch (Exception e) {
+ int errCode = 2106;
+ String msg = "Error while computing variance in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+ }
+ }
+
+ @Override
+ public void cleanup() {
+ intermediateSumSquare = null;
+ intermediateSum = null;
+ intermediateCount = null;
+ }
+
+ @Override
+ public Double getValue() {
+ Double var = null;
+ if (intermediateCount != null && intermediateCount > 0) {
+ Double avg = new Double(intermediateSum / intermediateCount);
+ Double avgSquare = new Double(intermediateSumSquare / intermediateCount);
+ var = avgSquare - avg*avg;
+ }
+ return var;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/WilsonBinConf.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/WilsonBinConf.java b/datafu-pig/src/main/java/datafu/pig/stats/WilsonBinConf.java
new file mode 100644
index 0000000..1448611
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/WilsonBinConf.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.commons.math.MathException;
+import org.apache.commons.math.distribution.NormalDistribution;
+import org.apache.commons.math.distribution.NormalDistributionImpl;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
+
+import datafu.pig.util.SimpleEvalFunc;
+
+/**
+ * Computes the {@link <a href="http://en.wikipedia.org/wiki/Binomial_proportion_confidence_interval#Wilson_score_interval" target="_blank">Wilsonian binomial proportion confidence interval</a>}
+ * <p>
+ * Constructor requires the confidence interval (alpha) parameter, and the
+ * parameters are the number of positive (success) outcomes and the total
+ * number of observations. The UDF returns the (lower,upper) confidence
+ * interval.
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * -- the Wilsonian binomial proportion confidence interval for scoring
+ * %declare WILSON_ALPHA 0.10
+ *
+ * define WilsonBinConf datafu.pig.stats.WilsonBinConf('$WILSON_ALPHA');
+ *
+ * bar = FOREACH foo GENERATE WilsonBinConf(successes, totals).lower as score;
+ * quux = ORDER bar BY score DESC;
+ * top = LIMIT quux 10;
+ * }
+ * </pre></p>
+ */
+public class WilsonBinConf extends SimpleEvalFunc<Tuple>
+{
+ private static TupleFactory tupleFactory = TupleFactory.getInstance();
+ private final double alpha;
+
+ public WilsonBinConf(double alpha)
+ {
+ this.alpha = alpha;
+ }
+
+ public WilsonBinConf(String alpha)
+ {
+ this(Double.parseDouble(alpha));
+ }
+
+ public Tuple call(Number x, Number n) throws IOException
+ {
+ if (x == null || n == null)
+ return null;
+ return binconf(x.longValue(), n.longValue());
+ }
+
+ /**
+ * @param x The number of positive (success) outcomes
+ * @param n The number of observations
+ * @return The (lower,upper) confidence interval
+ */
+ public Tuple binconf(Long x, Long n) throws IOException
+ {
+ NormalDistribution normalDist = new NormalDistributionImpl();
+
+ if (x == null || n == null)
+ return null;
+ if (x < 0 || n < 0)
+ throw new IllegalArgumentException("non-negative values expected");
+ if (x > n)
+ throw new IllegalArgumentException("invariant violation: number of successes > number of obs");
+ if (n == 0)
+ return tupleFactory.newTuple(Arrays.asList(Double.valueOf(0), Double.valueOf(0)));
+
+ try {
+ double zcrit = -1.0 * normalDist.inverseCumulativeProbability(alpha/2);
+ double z2 = zcrit * zcrit;
+ double p = x/(double)n;
+
+ double a = p + z2/2/n;
+ double b = zcrit * Math.sqrt((p * (1 - p) + z2/4/n)/n);
+ double c = (1 + z2/n);
+
+ double lower = (a - b) / c;
+ double upper = (a + b) / c;
+
+ // Add corrections for when x is very close to n. This improves the estimates.
+ // For more info on wilson binomial confidence interval, see paper:
+ // L.D. Brown, T.T. Cai and A. DasGupta, Interval estimation for a binomial proportion (with discussion),
+ // _Statistical Science,_*16*:101-133, 2001.
+ // http://www-stat.wharton.upenn.edu/~tcai/paper/Binomial-StatSci.pdf
+
+ if (x == 1)
+ lower = -Math.log(1 - alpha)/n;
+ if (x == (n - 1))
+ upper = 1 + Math.log(1 - alpha)/n;
+
+ return tupleFactory.newTuple(Arrays.asList(lower, upper));
+ }
+ catch (MathException e) {
+ throw new IOException("math error", e);
+ }
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try {
+ Schema innerSchema = new Schema(Arrays.asList(
+ new Schema.FieldSchema("lower", DataType.DOUBLE),
+ new Schema.FieldSchema("upper", DataType.DOUBLE)));
+
+ return new Schema(new FieldSchema(null, innerSchema, DataType.TUPLE));
+ } catch(FrontendException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/entropy/ChaoShenEntropyEstimator.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/entropy/ChaoShenEntropyEstimator.java b/datafu-pig/src/main/java/datafu/pig/stats/entropy/ChaoShenEntropyEstimator.java
new file mode 100644
index 0000000..bfb7398
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/entropy/ChaoShenEntropyEstimator.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats.entropy;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.backend.executionengine.ExecException;
+
+import java.util.Collections;
+
+
+
+/*
+ * This class implements the Chao-Shen entropy estimator based on this
+ * {@link <a href="http://www.researchgate.net/publication/226041346_Nonparametric_estimation_of_Shannons_index_of_diversity_when_there_are_unseen_species_in_sample/file/d912f510a79b8aae74.pdf" target="_blank">paper</a>}
+ * <p>
+ * It combines the {@link <a href="http://en.wikipedia.org/wiki/Horvitz%E2%80%93Thompson_estimator" target="_blank">Horvitz-Thompson estimator</a>}
+ * to adjust for missing species and the concept of sample coverage to estimate the relative abundances of species in the sample.
+ * The entropy calculation formula is as follows:
+ * <pre>
+ * {@code
+ * H(X) = - SUM( ( C * c(x) / N * log(C * c(x) / N) ) / ( 1 - (1 - C * c(x) / N) ^ N ) )
+ * c(x) is the occurrence frequency of sample x, N is the sum of c(x)
+ * C = 1 - N1 / N, N1 is the number of samples whose c(x) equals 1.
+ * }
+ * </pre>
+ * </p>
+ */
+class ChaoShenEntropyEstimator extends EntropyEstimator {
+ //sample frequency
+ private AccumulativeSampleFrequencyMap freqBaggedMap;
+
+ //sum of frequency of all samples
+ private long N;
+
+ //number of samples whose occurrence frequency is 1
+ private long N1;
+
+ ChaoShenEntropyEstimator(String base){
+ super(base);
+ reset();
+ }
+
+ @Override
+ public void accumulate(long cx) throws ExecException {
+ if(cx > 0) {
+ freqBaggedMap.accumulate(cx);
+ N += cx;
+ if(cx == 1) {
+ N1++;
+ }
+ }
+ }
+
+ @Override
+ public double getEntropy() {
+ double h = 0.0;
+
+ if(N > 0) {
+
+ if(N1 == N) {
+ //avoid c == 0
+ N1 = N - 1;
+ }
+
+ //sample coverage estimation
+ double c = 1 - (double)N1 / N;
+
+ try {
+ //from in-memory frequency map
+ for(Iterator<Map.Entry<Long, MutableLong>> iter = this.freqBaggedMap.getInternalMap().entrySet().iterator();
+ iter.hasNext(); ) {
+ Map.Entry<Long, MutableLong> entry = iter.next();
+ long cx = entry.getKey();
+ long cnt = entry.getValue().longValue();
+ h += accumlateEntropy(cx, this.N, c, cnt);
+ }
+ //from backup databag
+ for(Iterator<Tuple> iter = this.freqBaggedMap.getInternalBag().iterator();
+ iter.hasNext(); ) {
+ Tuple t = iter.next();
+ long cx = (Long)t.get(0);
+ long cnt = (Long)t.get(1);
+ h += accumlateEntropy(cx, this.N, c, cnt);
+ }
+ } catch(ExecException ex) {
+ throw new RuntimeException(
+ "Error while computing chao-shen entropy, exception: " + ex);
+ }
+ }
+
+ return EntropyUtil.logTransform(h, super.base);
+ }
+
+ private double accumlateEntropy(long cx,
+ long N,
+ double c,
+ long cnt) {
+ //sample proportion
+ double p = (double)cx / N;
+ //sample coverage adjusted probability
+ double pa = c * p;
+ //probability to see an individual in the sample
+ double la = 1 - Math.pow((1 - pa), N);
+ return -(cnt * pa * Math.log(pa) / la);
+ }
+
+ @Override
+ public void reset() {
+ this.N = 0;
+ this.N1 = 0;
+ if(this.freqBaggedMap != null) {
+ this.freqBaggedMap.clear();
+ }
+ this.freqBaggedMap = new AccumulativeSampleFrequencyMap();
+ }
+
+ /*
+ * A map backed by a data bag to record the sample occurrence frequencies which might be repeated
+ * The purpose of this class is that we suspect in the real applications, the sample occurrence frequency
+ * might be repeated and putting all these repeated numbers into databag may be space inefficient
+ * So we have an in-memory map to record the repetitions and use a databag to backup when the size of the
+ * in-memory map exceeds a pre-defined threshold. At present we use 5M as the default threshold to control
+ * the number of entries in the in-memory map, which approximates to 80M bytes.
+ */
+ private class AccumulativeSampleFrequencyMap {
+
+ private long spillBytesThreshold;
+
+ /* key is the sample occurrence frequency
+ * value is the number of repetitions of this frequency
+ */
+ private Map<Long, MutableLong> countMap;
+
+ /* the backed databag
+ * each tuple has 2 elements
+ * the 1st element is sample occurrence frequency
+ * the 2nd element is the number of repetitions
+ */
+ private DataBag countBag;
+
+ AccumulativeSampleFrequencyMap(){
+ this(1024 * 1024 * 5 * (Long.SIZE + Long.SIZE) / 8);
+ }
+
+ AccumulativeSampleFrequencyMap(long spillBytesThreshold) {
+ this.spillBytesThreshold = spillBytesThreshold;
+ clear();
+ }
+
+ void accumulate(long key) throws ExecException {
+ MutableLong val = this.countMap.get(key);
+ if(val == null) {
+ this.countMap.put(key, new MutableLong(1));
+ } else {
+ val.add(1);
+ }
+
+ if(this.countMap.size() * (Long.SIZE + Long.SIZE) / 8 > spillBytesThreshold) {
+ spillFromMap2Bag();
+ }
+ }
+
+ private void spillFromMap2Bag() throws ExecException {
+ for(Map.Entry<Long, MutableLong> entry : this.countMap.entrySet()) {
+ Tuple t = TupleFactory.getInstance().newTuple(2);
+ t.set(0, entry.getKey());
+ t.set(1, entry.getValue().longValue());
+ this.countBag.add(t);
+ }
+ this.countMap.clear();
+ }
+
+ Map<Long, MutableLong> getInternalMap() {
+ return Collections.unmodifiableMap(this.countMap);
+ }
+
+ DataBag getInternalBag() {
+ return this.countBag;
+ }
+
+ void clear() {
+ countMap = new HashMap<Long, MutableLong>();
+ countBag = BagFactory.getInstance().newDefaultBag();
+ }
+ }
+
+ private class MutableLong {
+ private long val;
+
+ MutableLong(){
+ this(0L);
+ }
+
+ MutableLong(long val) {
+ this.val = val;
+ }
+
+ long longValue(){
+ return val;
+ }
+
+ void add(long additive){
+ this.val += additive;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/entropy/CondEntropy.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/entropy/CondEntropy.java b/datafu-pig/src/main/java/datafu/pig/stats/entropy/CondEntropy.java
new file mode 100644
index 0000000..26b743e
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/entropy/CondEntropy.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats.entropy;
+
+import org.apache.pig.AccumulatorEvalFunc;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+
+
+/**
+ * Calculate conditional entropy H(Y|X) of random variables X and Y following conditional entropy's
+ * {@link <a href="http://en.wikipedia.org/wiki/Conditional_entropy" target="_blank">wiki definition</a>},
+ * X is the conditional variable and Y is the variable that conditions on X.
+ * <p>
+ * Each tuple of the input bag has 2 fields, the 1st field is an object instance of variable X and
+ * the 2nd field is an object instance of variable Y. An exception will be thrown if the number of fields is not 2.
+ * </p>
+ * <p>
+ * This UDF's constructor definition and parameters are the same as that of {@link datafu.pig.stats.entropy.Entropy}
+ * </p>
+ * <p>
+ * Note:
+ * <ul>
+ * <li>The input bag to this UDF must be <b>sorted</b> on X and Y, with X in the first sort order.
+ * An exception will be thrown if the input bag is not sorted.
+ * <li>The returned entropy value is of double type.
+ * </ul>
+ * </p>
+ * <p>
+ * How to use:
+ * </p>
+ * <p>
+ * This UDF calculates conditional entropy given raw data tuples of X and Y without the need to pre-compute per tuple occurrence frequency.
+ * </p>
+ * <p>
+ * It could be used in a nested FOREACH after a GROUP BY, in which we sort the inner bag and use the sorted bag as this UDF's input.
+ * </p>
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * --define empirical conditional entropy with Euler's number as the logarithm base
+ * define CondEntropy datafu.pig.stats.entropy.CondEntropy();
+ *
+ * input = LOAD 'input' AS (grp: chararray, valX: double, valY: double);
+ *
+ * -- calculate conditional entropy H(Y|X) in each group
+ * input_group_g = GROUP input BY grp;
+ * entropy_group = FOREACH input_group_g {
+ * input_val = input.(valX, valY)
+ * input_ordered = ORDER input_val BY $0, $1;
+ * GENERATE FLATTEN(group) AS group, CondEntropy(input_ordered) AS cond_entropy;
+ * }
+ * }
+ * </pre>
+ * </p>
+ * Use case to calculate mutual information:
+ * <p>
+ * <pre>
+ * {@code
+ * ------------
+ * -- calculate mutual information I(X, Y) using conditional entropy UDF and entropy UDF
+ * -- I(X, Y) = H(Y) - H(Y|X)
+ * ------------
+ *
+ * define CondEntropy datafu.pig.stats.entropy.CondEntropy();
+ * define Entropy datafu.pig.stats.entropy.Entropy();
+ *
+ * input = LOAD 'input' AS (grp: chararray, valX: double, valY: double);
+ *
+ * -- calculate the I(X,Y) in each group
+ * input_group_g = GROUP input BY grp;
+ * mutual_information = FOREACH input_group_g {
+ * input_val_x_y = input.(valX, valY);
+ * input_val_x_y_ordered = ORDER input_val_x_y BY $0,$1;
+ * input_val_y = input.valY;
+ * input_val_y_ordered = ORDER input_val_y BY $0;
+ * cond_h_x_y = CondEntropy(input_val_x_y_ordered);
+ * h_y = Entropy(input_val_y_ordered);
+ * GENERATE FLATTEN(group), h_y - cond_h_x_y;
+ * }
+ * }
+ * </pre>
+ * </p>
+ * @see Entropy
+ */
+public class CondEntropy extends AccumulatorEvalFunc<Double> {
+ //last visited tuple of <x,y>
+ private Tuple xy;
+
+ //number of occurrence of last visited <x,y>
+ private long cxy;
+
+ //number of occurrence of last visited x
+ private long cx;
+
+ //comparison result between the present tuple and the last visited tuple
+ private int lastCmp;
+
+ //entropy estimator for H(x,y)
+ private EntropyEstimator combEstimator;
+
+ //entropy estimator for H(x)
+ private EntropyEstimator condXEstimator;
+
+ public CondEntropy() throws ExecException
+ {
+ this(EntropyEstimator.EMPIRICAL_ESTIMATOR);
+ }
+
+ public CondEntropy(String type) throws ExecException
+ {
+ this(type, EntropyUtil.LOG);
+ }
+
+ public CondEntropy(String type, String base) throws ExecException
+ {
+ try {
+ this.combEstimator = EntropyEstimator.createEstimator(type, base);
+ this.condXEstimator = EntropyEstimator.createEstimator(type, base);
+ } catch (IllegalArgumentException ex) {
+ throw new ExecException(String.format(
+ "Fail to initialize StreamingCondEntropy with entropy estimator of type (%s), base: (%s). Exception: (%s)",
+ type, base, ex));
+ }
+ cleanup();
+ }
+
+ /*
+ * Accumulate occurrence frequency of <x,y> and x
+ * as we stream through the input bag
+ */
+ @Override
+ public void accumulate(Tuple input) throws IOException
+ {
+ for (Tuple t : (DataBag) input.get(0)) {
+
+ if (this.xy != null)
+ {
+ int cmp = t.compareTo(this.xy);
+
+ //check if the comparison result is different from previous compare result
+ if ((cmp < 0 && this.lastCmp > 0)
+ || (cmp > 0 && this.lastCmp < 0)) {
+ throw new ExecException("Out of order! previous tuple: " + this.xy + ", present tuple: " + t
+ + ", comparsion: " + cmp + ", previous comparsion: " + this.lastCmp);
+ }
+ if (cmp != 0) {
+ //different <x,y>
+ this.combEstimator.accumulate(this.cxy);
+ this.cxy = 0;
+ this.lastCmp = cmp;
+ if(DataType.compare(this.xy.get(0), t.get(0)) != 0) {
+ //different x
+ this.condXEstimator.accumulate(this.cx);
+ this.cx = 0;
+ }
+ }
+ }
+
+ //set tuple t as the next tuple for comparison
+ this.xy = t;
+
+ //accumulate cx
+ this.cx++;
+
+ //accumulate cxy
+ this.cxy++;
+ }
+ }
+
+ @Override
+ public Double getValue()
+ {
+ //do not miss the last tuple
+ try {
+ this.combEstimator.accumulate(this.cxy);
+ this.condXEstimator.accumulate(this.cx);
+ } catch (ExecException ex) {
+ throw new RuntimeException("Error while accumulating sample frequency: " + ex);
+ }
+
+ //Chain rule: H(Y|X) = H(X, Y) - H(X)
+ return this.combEstimator.getEntropy() - this.condXEstimator.getEntropy();
+ }
+
+ @Override
+ public void cleanup()
+ {
+ this.xy = null;
+ this.cxy = 0;
+ this.cx = 0;
+ this.lastCmp = 0;
+ this.combEstimator.reset();
+ this.condXEstimator.reset();
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try {
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.BAG)
+ {
+ throw new RuntimeException("Expected a BAG as input");
+ }
+
+ Schema inputBagSchema = inputFieldSchema.schema;
+
+ if (inputBagSchema.getField(0).type != DataType.TUPLE)
+ {
+ throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
+ DataType.findTypeName(inputBagSchema.getField(0).type)));
+ }
+
+ Schema tupleSchema = inputBagSchema.getField(0).schema;
+
+ if(tupleSchema == null) {
+ throw new RuntimeException("The tuple of the input bag has no schema");
+ }
+
+ List<Schema.FieldSchema> fieldSchemaList = tupleSchema.getFields();
+
+ if(fieldSchemaList == null || fieldSchemaList.size() != 2) {
+ throw new RuntimeException("The field schema of the input tuple is null or its size is not 2");
+ }
+
+ return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
+ .getName()
+ .toLowerCase(), input),
+ DataType.DOUBLE));
+ } catch (FrontendException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/entropy/EmpiricalCountEntropy.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/entropy/EmpiricalCountEntropy.java b/datafu-pig/src/main/java/datafu/pig/stats/entropy/EmpiricalCountEntropy.java
new file mode 100644
index 0000000..388b80f
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/entropy/EmpiricalCountEntropy.java
@@ -0,0 +1,425 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats.entropy;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.pig.AccumulatorEvalFunc;
+import org.apache.pig.Algebraic;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.PigException;
+import org.apache.pig.PigWarning;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+import datafu.pig.stats.entropy.EntropyUtil;
+
+
+/**
+ * Calculate the empirical entropy of random variable X given its occurrence frequencies, following entropy's
+ * {@link <a href="http://en.wikipedia.org/wiki/Entropy_%28information_theory%29" target="_blank">wiki definition</a>}
+ * <p>
+ * This UDF's constructor takes 1 argument: the logarithm base, whose definition is the same as that defined in {@link datafu.pig.stats.entropy.Entropy}
+ * </p>
+ * <p>
+ * Note:
+ * <ul>
+ * <li>Unlike {@link datafu.pig.stats.entropy.Entropy}, which calculates entropy from sorted raw data bag in accumulative mode,
+ * this UDF calculates entropy from the data's occurrence frequencies which does not need to be sorted, either in accumulative or algebraic mode.
+ * <li>Each tuple of the UDF's input bag <b>must only</b> have 1 field, the occurrence frequency of a data instance,
+ * and the data type of this field <b>must</b> be int or long. Otherwise, an exception will be thrown.
+ * <li>Negative frequency number will be silently discarded and a warning message will be logged in the job's log file.
+ * <li>The returned entropy value is of double type.
+ * </ul>
+ * </p>
+ * <p>
+ * How to use:
+ * </p>
+ * <p>
+ * To use this UDF, customer needs to pre-compute the occurrence frequency of each data instance, often in an outer GROUP BY
+ * , and then use this UDF to calculate entropy with those frequency numbers in another outer GROUP BY.
+ * </p>
+ * <p>
+ * Compared with {@link datafu.pig.stats.entropy.Entropy}, this UDF is more scalable when we need to handle a very large data set,
+ * since it could distribute computation onto mappers and take advantage of combiners to reduce intermedidate output from mappers to reducers.
+ * </p>
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ *
+ * define Entropy datafu.pig.stats.entropy.EmpiricalCountEntropy();
+ *
+ * input = LOAD 'input' AS (val: double);
+ *
+ * -- calculate the occurrence of each instance
+ * counts_g = GROUP input BY val;
+ * counts = FOREACh counts_g GENERATE COUNT(input) AS cnt;
+ *
+ * -- calculate entropy
+ * input_counts_g = GROUP counts ALL;
+ * entropy = FOREACH input_counts_g GENERATE Entropy(counts) AS entropy;
+ * }
+ * </pre>
+ * </p>
+ * Use case to calculate mutual information using EmpiricalCountEntropy:
+ * <p>
+ * <pre>
+ * {@code
+ *
+ * define Entropy datafu.pig.stats.entropy.EmpiricalCountEntropy();
+ *
+ * input = LOAD 'input' AS (valX: double, valY: double);
+ *
+ * ------------
+ * -- calculate mutual information I(X, Y) using entropy
+ * -- I(X, Y) = H(X) + H(Y) - H(X, Y)
+ * ------------
+ *
+ * input_x_y_g = GROUP input BY (valX, valY);
+ * input_x_y_cnt = FOREACH input_x_y_g GENERATE flatten(group) as (valX, valY), COUNT(input) AS cnt;
+ *
+ * input_x_g = GROUP input_x_y_cnt BY valX;
+ * input_x_cnt = FOREACH input_x_g GENERATE flatten(group) as valX, SUM(input_x_y_cnt.cnt) AS cnt;
+ *
+ * input_y_g = GROUP input_x_y_cnt BY valY;
+ * input_y_cnt = FOREACH input_y_g GENERATE flatten(group) as valY, SUM(input_x_y_cnt.cnt) AS cnt;
+ *
+ * input_x_y_entropy_g = GROUP input_x_y_cnt ALL;
+ * input_x_y_entropy = FOREACH input_x_y_entropy_g {
+ * input_x_y_entropy_cnt = input_x_y_cnt.cnt;
+ * GENERATE Entropy(input_x_y_entropy_cnt) AS x_y_entropy;
+ * }
+ *
+ * input_x_entropy_g = GROUP input_x_cnt ALL;
+ * input_x_entropy = FOREACH input_x_entropy_g {
+ * input_x_entropy_cnt = input_x_cnt.cnt;
+ * GENERATE Entropy(input_x_entropy_cnt) AS x_entropy;
+ * }
+ *
+ * input_y_entropy_g = GROUP input_y_cnt ALL;
+ * input_y_entropy = FOREACH input_y_entropy_g {
+ * input_y_entropy_cnt = input_y_cnt.cnt;
+ * GENERATE Entropy(input_y_entropy_cnt) AS y_entropy;
+ * }
+ *
+ * input_mi_cross = CROSS input_x_y_entropy, input_x_entropy, input_y_entropy;
+ * input_mi = FOREACH input_mi_cross GENERATE (input_x_entropy::x_entropy +
+ * input_y_entropy::y_entropy -
+ * input_x_y_entropy::x_y_entropy) AS mi;
+ * }
+ * </pre>
+ * </p>
+ * @see Entropy
+ */
+
+public class EmpiricalCountEntropy extends AccumulatorEvalFunc<Double> implements Algebraic {
+
+ private static TupleFactory mTupleFactory = TupleFactory.getInstance();
+
+ //entropy estimator for accumulator
+ //re-use the same entropy estimator for StreamingEntropy
+ private EntropyEstimator streamEstimator;
+
+ //logarithm base
+ private String base;
+
+ public EmpiricalCountEntropy() throws ExecException {
+ //empirical estimator using Euler's number as logarithm base
+ this(EntropyUtil.LOG);
+ }
+
+ public EmpiricalCountEntropy(String base) throws ExecException {
+ try {
+ this.streamEstimator = EntropyEstimator.createEstimator(EntropyEstimator.EMPIRICAL_ESTIMATOR, base);
+ } catch (IllegalArgumentException ex) {
+ throw new ExecException(
+ String.format("Fail to initialize EmpiricalCountEntropy with logarithm base: (%s), exception: (%s)", base, ex));
+ }
+ this.base = base;
+ }
+
+ /*
+ * Algebraic implementation part
+ */
+
+ private String param = null;
+ private String getParam()
+ {
+ if (param == null) {
+ if (this.base != null) {
+ param = String.format("('%s')", this.base);
+ } else {
+ param = "";
+ }
+ }
+ return param;
+ }
+
+ @Override
+ public String getFinal() {
+ return Final.class.getName() + getParam();
+ }
+
+ @Override
+ public String getInitial() {
+ return Initial.class.getName() + getParam();
+ }
+
+ @Override
+ public String getIntermed() {
+ return Intermediate.class.getName() + getParam();
+ }
+
+ static public class Initial extends EvalFunc<Tuple> {
+
+ public Initial(){}
+
+ public Initial(String base){}
+
+ @Override
+ public Tuple exec(Tuple input) throws IOException {
+ Tuple t = mTupleFactory.newTuple(2);
+
+ try{
+ //input is a bag with one tuple containing
+ //the sample's occurrence frequency
+ DataBag bg = (DataBag) input.get(0);
+ Long cxl = null;
+ if(bg.iterator().hasNext()) {
+ Tuple tp = bg.iterator().next();
+ cxl = ((Number)(tp.get(0))).longValue();
+ }
+
+ if(cxl == null || cxl.longValue() < 0) {
+ //emit null in case of invalid input frequency
+ t.set(0, null);
+ t.set(1, null);
+ warn("Non-positive input frequency number: " + cxl, PigWarning.UDF_WARNING_1);
+ } else {
+ long cx = cxl.longValue();
+ double logcx = (cx > 0 ? Math.log(cx) : 0);
+ double cxlogcx = cx * logcx;
+
+ //1st element of the returned tuple is freq * log(freq)
+ t.set(0, cxlogcx);
+
+ //2nd element of the returned tuple is freq
+ t.set(1, cxl);
+ }
+
+ return t;
+ } catch (ExecException ee) {
+ throw ee;
+ } catch(Exception e) {
+ int errCode = 10080;
+ String msg = "Error while computing entropy in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+ }
+
+ }
+
+ }
+
+ static public class Intermediate extends EvalFunc<Tuple> {
+
+ public Intermediate(){}
+
+ public Intermediate(String base){}
+
+ @Override
+ public Tuple exec(Tuple input) throws IOException {
+ try {
+ DataBag b = (DataBag)input.get(0);
+ return combine(b);
+ } catch (ExecException ee) {
+ throw ee;
+ } catch (Exception e) {
+ int errCode = 10081;
+ String msg = "Error while computing entropy in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+ }
+ }
+ }
+
+ static public class Final extends EvalFunc<Double> {
+ private String base;
+
+ public Final()
+ {
+ this(EntropyUtil.LOG);
+ }
+
+ public Final(String base)
+ {
+ this.base = base;
+ }
+
+ @Override
+ public Double exec(Tuple input) throws IOException {
+ try {
+ DataBag b = (DataBag)input.get(0);
+ Tuple combined = combine(b);
+
+ Double sumOfCxLogCx = (Double)combined.get(0);
+ Long sumOfCx = (Long)combined.get(1);
+
+ if(sumOfCxLogCx == null || sumOfCx == null) {
+ //emit null if there is at least 1 invalid input
+ warn("Invalid null field output from combine(), " +
+ "1st field: " + sumOfCxLogCx + ", 2nd field: " + sumOfCx, PigWarning.UDF_WARNING_1);
+ return null;
+ }
+
+ Double entropy = null;
+
+ double scxlogcx = sumOfCxLogCx.doubleValue();
+ long scx = sumOfCx.longValue();
+
+ if (scx > 0) {
+ //H(X) = log(N) - 1 / N * SUM(c(x) * log(c(x)) )
+ entropy = EntropyUtil.logTransform(Math.log(scx) - scxlogcx / scx, this.base);
+ }
+
+ return entropy;
+ } catch (ExecException ee) {
+ throw ee;
+ } catch (Exception e) {
+ int errCode = 10082;
+ String msg = "Error while computing average in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+ }
+
+ }
+ }
+
+ static protected Tuple combine(DataBag values) throws ExecException {
+ Tuple output = mTupleFactory.newTuple(2);
+
+ boolean sawNonNull = false;
+ double sumOfCxLogCx = 0;
+ long sumOfCx = 0;
+
+ for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
+ Tuple t = it.next();
+ Double scxlogcx = (Double)t.get(0);
+ Long scx = (Long)t.get(1);
+
+ if(scxlogcx != null && scx != null) {
+ sumOfCxLogCx += scxlogcx.doubleValue();
+ sumOfCx += scx.longValue();
+ sawNonNull = true;
+ }
+ }
+
+ if (sawNonNull) {
+ output.set(0, sumOfCxLogCx);
+ output.set(1, sumOfCx);
+ } else {
+ //emit null if there is no invalid input
+ output.set(0, null);
+ output.set(1, null);
+ }
+
+ return output;
+ }
+
+ /*
+ * Accumulator implementation part
+ */
+
+ @Override
+ public void accumulate(Tuple input) throws IOException
+ {
+ for (Tuple t : (DataBag) input.get(0)) {
+ long cx = ((Number)(t.get(0))).longValue();
+ this.streamEstimator.accumulate(cx);
+ }
+ }
+
+ @Override
+ public Double getValue()
+ {
+ return this.streamEstimator.getEntropy();
+ }
+
+ @Override
+ public void cleanup()
+ {
+ this.streamEstimator.reset();
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try {
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.BAG)
+ {
+ throw new RuntimeException("Expected a BAG as input");
+ }
+
+ Schema inputBagSchema = inputFieldSchema.schema;
+
+ if (inputBagSchema.getField(0).type != DataType.TUPLE)
+ {
+ throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
+ DataType.findTypeName(inputBagSchema.getField(0).type)));
+ }
+
+ Schema tupleSchema = inputBagSchema.getField(0).schema;
+
+ if(tupleSchema == null) {
+ throw new RuntimeException("The tuple of input bag has no schema");
+ }
+
+ List<Schema.FieldSchema> fieldSchemaList = tupleSchema.getFields();
+
+ if(fieldSchemaList == null || fieldSchemaList.size() != 1) {
+ throw new RuntimeException("The field schema of the input tuple is null or its size is not 1");
+ }
+
+ if(fieldSchemaList.get(0).type != DataType.INTEGER &&
+ fieldSchemaList.get(0).type != DataType.LONG )
+ {
+ String[] expectedTypes = new String[] {DataType.findTypeName(DataType.INTEGER),
+ DataType.findTypeName(DataType.LONG)};
+ throw new RuntimeException("Expect the type of the input tuple to be of (" +
+ java.util.Arrays.toString(expectedTypes) + "), but instead found " +
+ DataType.findTypeName(fieldSchemaList.get(0).type));
+ }
+
+ return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
+ .getName()
+ .toLowerCase(), input),
+ DataType.DOUBLE));
+ } catch (FrontendException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/entropy/EmpiricalEntropyEstimator.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/entropy/EmpiricalEntropyEstimator.java b/datafu-pig/src/main/java/datafu/pig/stats/entropy/EmpiricalEntropyEstimator.java
new file mode 100644
index 0000000..3f0acab
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/entropy/EmpiricalEntropyEstimator.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats.entropy;
+
+/*
+ * This entropy estimator calculates the empirical entropy of given species
+ * using the occurrence frequency of individual in the sample as an estimation of its probability.
+ * The empirical estimator is also called the maximum likelihood estimator (MLE).
+ * <p>
+ * It applies the following formula to compute entropy:
+ * <pre>
+ * {@code
+ * H(X) = log(N) - 1 / N * SUM(c(x) * log(c(x)) )
+ * c(x) is the occurrence frequency of sample x, N is the sum of c(x)
+ * }
+ * </pre>
+ * </p>
+ * <p>
+ * This entropy estimator is widely used when the number of species is known and small.
+ * But it is biased since it does not cover the rare species with zero frequency in the sample.
+ * </p>
+ */
+class EmpiricalEntropyEstimator extends EntropyEstimator {
+
+ //sum of frequency cx of all input samples
+ private long N;
+
+ //sum of cx * log(cx) of all input samples
+ private double M;
+
+ EmpiricalEntropyEstimator(String base) throws IllegalArgumentException {
+ super(base);
+ reset();
+ }
+
+ @Override
+ public void accumulate(long cx) {
+ if(cx > 0) {
+ N += cx;
+ M += cx * Math.log(cx);
+ }
+ }
+
+ @Override
+ public double getEntropy() {
+ return N > 0 ? EntropyUtil.logTransform(Math.log(N) - M / N, super.base) : 0;
+ }
+
+ @Override
+ public void reset(){
+ N = 0;
+ M = 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/entropy/Entropy.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/entropy/Entropy.java b/datafu-pig/src/main/java/datafu/pig/stats/entropy/Entropy.java
new file mode 100644
index 0000000..9dfff1a
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/entropy/Entropy.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats.entropy;
+
+import java.io.IOException;
+
+import org.apache.pig.AccumulatorEvalFunc;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+
+
+/**
+ * Calculate entropy H(X) of random variable X following entropy's
+ * {@link <a href="http://en.wikipedia.org/wiki/Entropy_%28information_theory%29" target="_blank">wiki definition</a>}
+ * <p>
+ * This UDF's constructor takes 2 arguments.
+ * </p>
+ * <p>
+ * The 1st argument, the type of entropy estimator algorithm we currently support, includes:
+ * <ul>
+ * <li>empirical (empirical entropy estimator)
+ * <li>chaosh (Chao-Shen entropy estimator)
+ * </ul>
+ * </p>
+ * <p>
+ * The default estimation algorithm is empirical.
+ * </p>
+ * <p>
+ * The 2nd argument, the logarithm base we currently support, includes:
+ * </p>
+ * <p>
+ * <ul>
+ * <li>log (use Euler's number as the logarithm base)
+ * <li>log2 (use 2 as the logarithm base)
+ * <li>log10 (use 10 as the logarithm base)
+ * </ul>
+ * </p>
+ * <p>
+ * The default logarithm base is log.
+ * </p>
+ * <p>
+ * Note:
+ * <ul>
+ * <li>The input to this UDF must be a <b>sorted</b> bag of raw data tuples of X.
+ * An exception will be thrown if the input bag is not sorted
+ * <li>The returned entropy value is of double type.
+ * </ul>
+ * </p>
+ * <p>
+ * How to use:
+ * </p>
+ * <p>
+ * This UDF calculates entropy from raw data tuples without the need to pre-compute per tuple occurrence frequency.
+ * </p>
+ * <p>
+ * It could be used in a nested FOREACH after a GROUP BY, in which we sort the inner bag and use the sorted bag as this UDF's input.
+ * </p>
+ * Example:
+ * <p>
+ * <pre>
+ * {@code
+ * --calculate empirical entropy with Euler's number as the logarithm base
+ * define Entropy datafu.pig.stats.entropy.Entropy();
+ *
+ * input = LOAD 'input' AS (grp: chararray, val: double);
+ *
+ * -- calculate the input's entropy in each group
+ * input_group_g = GROUP input BY grp;
+ * entropy_group = FOREACH input_group_g {
+ * input_val = input.val;
+ * input_ordered = ORDER input_val BY $0;
+ * GENERATE FLATTEN(group) AS group, Entropy(input_ordered) AS entropy;
+ * }
+ * }
+ * </pre>
+ * </p>
+ * @see CondEntropy
+ * @see EmpiricalCountEntropy
+ */
+public class Entropy extends AccumulatorEvalFunc<Double>
+{
+ //last visited tuple
+ private Tuple x;
+
+ //number of occurrence of last visited tuple
+ private long cx;
+
+ //comparison result between the present tuple and the last visited tuple
+ private int lastCmp;
+
+ //entropy estimator that accumulates sample's occurrence frequency to
+ //calculates the actual entropy
+ private EntropyEstimator estimator;
+
+ public Entropy() throws ExecException
+ {
+ this(EntropyEstimator.EMPIRICAL_ESTIMATOR);
+ }
+
+ public Entropy(String type) throws ExecException
+ {
+ this(type, EntropyUtil.LOG);
+ }
+
+ public Entropy(String type, String base) throws ExecException
+ {
+ try {
+ this.estimator = EntropyEstimator.createEstimator(type, base);
+ } catch (IllegalArgumentException ex) {
+ throw new ExecException(
+ String.format("Fail to initialize StreamingEntropy with entropy estimator of type (%s), base: (%s), exception: (%s)",
+ type, base, ex)
+ );
+ }
+ cleanup();
+ }
+
+ /*
+ * Accumulate occurrence frequency of each tuple as we stream through the input bag
+ */
+ @Override
+ public void accumulate(Tuple input) throws IOException
+ {
+ for (Tuple t : (DataBag) input.get(0)) {
+
+ if (this.x != null)
+ {
+ int cmp = t.compareTo(this.x);
+
+ //check if the comparison result is different from previous compare result
+ if ((cmp < 0 && this.lastCmp > 0)
+ || (cmp > 0 && this.lastCmp < 0)) {
+ throw new ExecException("Out of order! previous tuple: " + this.x + ", present tuple: " + t
+ + ", comparsion: " + cmp + ", previous comparsion: " + this.lastCmp);
+ }
+
+ if (cmp != 0) {
+ //different tuple
+ this.estimator.accumulate(this.cx);
+ this.cx = 0;
+ this.lastCmp = cmp;
+ }
+ }
+
+ //set tuple t as the next tuple for comparison
+ this.x = t;
+
+ //accumulate cx
+ this.cx++;
+ }
+ }
+
+ @Override
+ public Double getValue()
+ {
+ //do not miss the last tuple
+ try {
+ this.estimator.accumulate(this.cx);
+ } catch (ExecException ex) {
+ throw new RuntimeException("Error while accumulating sample frequency: " + ex);
+ }
+
+ return this.estimator.getEntropy();
+ }
+
+ @Override
+ public void cleanup()
+ {
+ this.x = null;
+ this.cx = 0;
+ this.lastCmp = 0;
+ this.estimator.reset();
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try {
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.BAG)
+ {
+ throw new RuntimeException("Expected a BAG as input");
+ }
+
+ Schema inputBagSchema = inputFieldSchema.schema;
+
+ if (inputBagSchema.getField(0).type != DataType.TUPLE)
+ {
+ throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
+ DataType.findTypeName(inputBagSchema.getField(0).type)));
+ }
+
+ return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
+ .getName()
+ .toLowerCase(), input),
+ DataType.DOUBLE));
+ } catch (FrontendException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/entropy/EntropyEstimator.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/entropy/EntropyEstimator.java b/datafu-pig/src/main/java/datafu/pig/stats/entropy/EntropyEstimator.java
new file mode 100644
index 0000000..336ef86
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/entropy/EntropyEstimator.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats.entropy;
+
+import org.apache.pig.backend.executionengine.ExecException;
+
+/*
+* Entropy estimator which exposes a unified interface to application
+* and hides the implementation details in its subclasses
+*/
+public abstract class EntropyEstimator {
+
+ public static final String EMPIRICAL_ESTIMATOR = "empirical";
+
+ public static final String CHAOSHEN_ESTIMATOR = "chaosh";
+
+ /*
+ * logarithm base
+ * by default, we use Euler's number as the default logarithm base
+ */
+ protected String base;
+
+ protected EntropyEstimator(String base) throws IllegalArgumentException {
+ if(!EntropyUtil.isValidLogBase(base)) {
+ throw new IllegalArgumentException("Invalid input logarithm base. " +
+ "Please refer to StreamingEntropy's javadoc for supported logarithm base");
+ }
+ this.base = base;
+ }
+
+ /*
+ * create estimator instance based on the input type
+ * @param type type of entropy estimator
+ * @param base logarithm base
+ * if the method is not supported, a null estimator instance is returned
+ * and the external application needs to handle this case
+ */
+ public static EntropyEstimator createEstimator(String type,
+ String base) throws IllegalArgumentException
+ {
+ //empirical estimator
+ if(EmpiricalEntropyEstimator.EMPIRICAL_ESTIMATOR.equalsIgnoreCase(type)) {
+ return new EmpiricalEntropyEstimator(base);
+ }
+ //chao-shen estimator
+ if(EmpiricalEntropyEstimator.CHAOSHEN_ESTIMATOR.equalsIgnoreCase(type)) {
+ return new ChaoShenEntropyEstimator(base);
+ }
+ //unsupported estimator type
+ throw new IllegalArgumentException("invalid input entropy estimator type. " +
+ "Please refer to StreamingEntropy's javadoc for the supported estimator types");
+ }
+
+ /*
+ * accumulate occurrence frequency from input stream of samples
+ * @param cx the occurrence frequency of the last input sample
+ */
+ public abstract void accumulate(long cx) throws ExecException;
+
+ /*
+ * calculate the output entropy value
+ * return entropy value as a double type number
+ */
+ public abstract double getEntropy();
+
+ /*
+ * cleanup and reset internal states
+ */
+ public abstract void reset();
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/entropy/EntropyUtil.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/entropy/EntropyUtil.java b/datafu-pig/src/main/java/datafu/pig/stats/entropy/EntropyUtil.java
new file mode 100644
index 0000000..685f3e2
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/entropy/EntropyUtil.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats.entropy;
+
+public class EntropyUtil {
+
+ public static final String LOG = "log";
+
+ public static final String LOG2 = "log2";
+
+ public static final String LOG10 = "log10";
+
+ /*
+ * Transform the input entropy to that in the input logarithm base
+ * The input entropy is assumed to be calculated in the Euler's number logarithm base
+ * */
+ public static double logTransform(double h, String base) {
+ if(LOG2.equalsIgnoreCase(base)) {
+ return h / Math.log(2);
+ }
+
+ if(LOG10.equalsIgnoreCase(base)) {
+ return h / Math.log(10);
+ }
+
+ return h;
+ }
+
+ public static boolean isValidLogBase(String base) {
+ return LOG.equalsIgnoreCase(base) ||
+ LOG2.equalsIgnoreCase(base) ||
+ LOG10.equalsIgnoreCase(base);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/package-info.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/package-info.java b/datafu-pig/src/main/java/datafu/pig/stats/package-info.java
new file mode 100644
index 0000000..4c100e7
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Statistics UDFs for computing median, quantiles, variance, confidence intervals, etc.
+ */
+package datafu.pig.stats;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/text/opennlp/CachedFile.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/text/opennlp/CachedFile.java b/datafu-pig/src/main/java/datafu/pig/text/opennlp/CachedFile.java
new file mode 100644
index 0000000..5832c81
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/text/opennlp/CachedFile.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package datafu.pig.text.opennlp;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class CachedFile {
+
+ public static String getFileName(String modelLink, String modelFile) throws IOException {
+ // if the symlink exists, use it, if not, use the raw name if it exists
+ // note: this is to help with testing, as it seems distributed cache doesn't work with PigUnit
+ String loadFile = modelFile;
+ if (!new File(loadFile).exists()) {
+ if (new File(modelLink).exists()) {
+ loadFile = modelLink;
+ } else {
+ throw new IOException(String.format("Could not load model, neither symlink %s nor file %s exist", modelFile, modelLink));
+ }
+ }
+ return loadFile;
+ }
+}
[05/19] DATAFU-27 Migrate build system to Gradle
Posted by wv...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/package-info.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/package-info.java b/src/java/datafu/pig/stats/package-info.java
deleted file mode 100644
index 4c100e7..0000000
--- a/src/java/datafu/pig/stats/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Statistics UDFs for computing median, quantiles, variance, confidence intervals, etc.
- */
-package datafu.pig.stats;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/text/opennlp/CachedFile.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/text/opennlp/CachedFile.java b/src/java/datafu/pig/text/opennlp/CachedFile.java
deleted file mode 100644
index 5832c81..0000000
--- a/src/java/datafu/pig/text/opennlp/CachedFile.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package datafu.pig.text.opennlp;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-public class CachedFile {
-
- public static String getFileName(String modelLink, String modelFile) throws IOException {
- // if the symlink exists, use it, if not, use the raw name if it exists
- // note: this is to help with testing, as it seems distributed cache doesn't work with PigUnit
- String loadFile = modelFile;
- if (!new File(loadFile).exists()) {
- if (new File(modelLink).exists()) {
- loadFile = modelLink;
- } else {
- throw new IOException(String.format("Could not load model, neither symlink %s nor file %s exist", modelFile, modelLink));
- }
- }
- return loadFile;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/text/opennlp/POSTag.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/text/opennlp/POSTag.java b/src/java/datafu/pig/text/opennlp/POSTag.java
deleted file mode 100644
index fb17c63..0000000
--- a/src/java/datafu/pig/text/opennlp/POSTag.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package datafu.pig.text.opennlp;
-
-import java.io.*;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import opennlp.tools.postag.POSModel;
-import opennlp.tools.postag.POSTaggerME;
-import org.apache.pig.EvalFunc;
-import org.apache.pig.data.*;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * The OpenNLP POSTag UDF tags bags of sequential words with parts of speech and confidence levels using the OpenNLP
- * toolset, and specifically the POSTaggerME class.
- * <p>
- * Example:
- * <pre>
- * {@code
- * define TokenizeME datafu.pig.text.opennlp.TokenizeME('data/en-token.bin');
- * define POSTag datafu.pig.text.opennlp.POSTag('data/en-pos-maxent.bin');
- *
- * -- input:
- * -- (Appetizers during happy hour range from low to high.)
- * input = LOAD 'input' AS (text:chararray);
- * --
- * -- ({(Appetizers),(during),(happy),(hour),(range),(from),(low),(to),(high),(.)})
- * tokenized = FOREACH input GENERATE TokenizeME(text) AS tokens;
- * --
- * -- output:
- * -- Tuple schema is: (word, tag, confidence)
- * -- ({(Appetizers,NNP,0.3619277937390988),(during,IN,0.7945543860326094),(happy,JJ,0.9888504792754391),
- * -- (hour,NN,0.9427455123502427),(range,NN,0.7335527963654751),(from,IN,0.9911576465589752),(low,JJ,0.9652034031895174),
- * -- (to,IN,0.7005347487371849),(high,JJ,0.8227771746247106),(.,.,0.9900983495480891)})
- * output = FOREACH tokenized GENERATE POSTag(tokens) AS tagged;
- * }
- * </pre>
- */
-public class POSTag extends EvalFunc<DataBag>
-{
- private POSTaggerME tagger = null;
- private static final String MODEL_FILE = "pos";
- private TupleFactory tf = TupleFactory.getInstance();
- private BagFactory bf = BagFactory.getInstance();
- private String modelPath;
-
- public POSTag(String modelPath) {
- this.modelPath = modelPath;
- }
-
- @Override
- public List<String> getCacheFiles() {
- List<String> list = new ArrayList<String>(1);
- list.add(this.modelPath + "#" + MODEL_FILE);
- return list;
- }
-
- // Enable multiple languages by specifying the model path. See http://text.sourceforge.net/models-1.5/
- public DataBag exec(Tuple input) throws IOException
- {
- DataBag inputBag = null;
-
- if(input.size() != 1) {
- throw new IOException();
- }
-
- inputBag = (DataBag)input.get(0);
- DataBag outBag = bf.newDefaultBag();
- if(this.tagger == null) {
- String loadFile = CachedFile.getFileName(MODEL_FILE, this.modelPath);
- InputStream modelIn = new FileInputStream(loadFile);
- InputStream buffer = new BufferedInputStream(modelIn);
- POSModel model = new POSModel(buffer);
- this.tagger = new POSTaggerME(model);
- }
-
- // Form an inputString array thing for tagger to act on
- int bagLength = (int)inputBag.size();
- String[] words = new String[bagLength];
-
- Iterator<Tuple> itr = inputBag.iterator();
- int i = 0;
- while(itr.hasNext()) {
- words[i] = (String)itr.next().get(0);
- i++;
- }
-
- // Compute tags and their probabilities
- String tags[] = this.tagger.tag(words);
- double probs[] = this.tagger.probs();
-
- // Build output bag of 3-tuples
- for(int j = 0; j < tags.length; j++) {
- Tuple newTuple = tf.newTuple(3);
- newTuple.set(0, words[j]);
- newTuple.set(1, tags[j]);
- newTuple.set(2, probs[j]);
- outBag.add(newTuple);
- }
-
- return outBag;
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try
- {
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.BAG)
- {
- throw new RuntimeException("Expected a BAG as input");
- }
-
- Schema inputBagSchema = inputFieldSchema.schema;
-
- if(inputBagSchema == null) {
- return null;
- }
-
- if (inputBagSchema.getField(0).type != DataType.TUPLE)
- {
- throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
- DataType.findTypeName(inputBagSchema.getField(0).type)));
- }
-
- Schema inputTupleSchema = inputBagSchema.getField(0).schema;
-
- if (inputTupleSchema.size() != 1)
- {
- throw new RuntimeException("Expected one field for the token data");
- }
-
- if (inputTupleSchema.getField(0).type != DataType.CHARARRAY)
- {
- throw new RuntimeException(String.format("Expected source to be a CHARARRAY, but instead found %s",
- DataType.findTypeName(inputTupleSchema.getField(0).type)));
- }
-
- Schema tupleSchema = new Schema();
- tupleSchema.add(new Schema.FieldSchema("token",DataType.CHARARRAY));
- tupleSchema.add(new Schema.FieldSchema("tag",DataType.CHARARRAY));
- tupleSchema.add(new Schema.FieldSchema("probability",DataType.DOUBLE));
-
- return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
- .getName()
- .toLowerCase(), input),
- tupleSchema,
- DataType.BAG));
- }
- catch (FrontendException e)
- {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/text/opennlp/SentenceDetect.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/text/opennlp/SentenceDetect.java b/src/java/datafu/pig/text/opennlp/SentenceDetect.java
deleted file mode 100644
index 50537fd..0000000
--- a/src/java/datafu/pig/text/opennlp/SentenceDetect.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package datafu.pig.text.opennlp;
-
-import java.io.*;
-import java.util.ArrayList;
-import java.util.List;
-
-import opennlp.tools.sentdetect.SentenceDetectorME;
-import opennlp.tools.sentdetect.SentenceModel;
-import org.apache.pig.EvalFunc;
-import org.apache.pig.data.*;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * The OpenNLP SentenceDectectors segment an input paragraph into sentences.
- * <p>
- * Example:
- * <pre>
- * {@code
- * define SentenceDetect datafu.pig.text.SentenceDetect('data/en-sent.bin');
- *
- * -- input:
- * -- ("I believe the Masons have infiltrated the Apache PMC. I believe laser beams control cat brains.")
- * infoo = LOAD 'input' AS (text:chararray);
-
- * -- output:
- * -- ({(I believe the Masons have infiltrated the Apache PMC.)(I believe laser beams control cat brains.)})
- * outfoo = FOREACH input GENERATE SentenceDetect(text) as sentences;
- * }
- * </pre>
- */
-public class SentenceDetect extends EvalFunc<DataBag>
-{
- private SentenceDetectorME sdetector = null;
- private static final String MODEL_FILE = "sentences";
- private TupleFactory tf = TupleFactory.getInstance();
- private BagFactory bf = BagFactory.getInstance();
- private String modelPath = null;
-
- public SentenceDetect(String modelPath) {
- this.modelPath = modelPath;
- }
-
- @Override
- public List<String> getCacheFiles() {
- List<String> list = new ArrayList<String>(1);
- list.add(this.modelPath + "#" + MODEL_FILE);
- return list;
- }
-
- // Enable multiple languages by specifying the model path. See http://text.sourceforge.net/models-1.5/
- public DataBag exec(Tuple input) throws IOException
- {
- if(input.size() != 1) {
- throw new IOException();
- }
-
- String inputString = input.get(0).toString();
- if(inputString == null || inputString == "") {
- return null;
- }
- DataBag outBag = bf.newDefaultBag();
- if(sdetector == null) {
- String loadFile = CachedFile.getFileName(MODEL_FILE, this.modelPath);
- InputStream is = new FileInputStream(modelPath);
- InputStream buffer = new BufferedInputStream(is);
- SentenceModel model = new SentenceModel(buffer);
- this.sdetector = new SentenceDetectorME(model);
- }
- String sentences[] = this.sdetector.sentDetect(inputString);
- for(String sentence : sentences) {
- Tuple outTuple = tf.newTuple(sentence);
- outBag.add(outTuple);
- }
- return outBag;
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try
- {
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.CHARARRAY)
- {
- throw new RuntimeException("Expected a CHARARRAY as input, but got a " + inputFieldSchema.toString());
- }
-
- Schema tupleSchema = new Schema();
- tupleSchema.add(new Schema.FieldSchema("sentence",DataType.CHARARRAY));
-
- return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
- .getName()
- .toLowerCase(), input),
- tupleSchema,
- DataType.BAG));
- }
- catch (FrontendException e)
- {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/text/opennlp/TokenizeME.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/text/opennlp/TokenizeME.java b/src/java/datafu/pig/text/opennlp/TokenizeME.java
deleted file mode 100644
index f1f4257..0000000
--- a/src/java/datafu/pig/text/opennlp/TokenizeME.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package datafu.pig.text.opennlp;
-
-import java.io.*;
-import java.util.ArrayList;
-import java.util.List;
-
-import opennlp.tools.tokenize.TokenizerME;
-import opennlp.tools.tokenize.TokenizerModel;
-import org.apache.pig.EvalFunc;
-import org.apache.pig.data.*;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * The OpenNLP Tokenizers segment an input character sequence into tokens using the OpenNLP TokenizeME class, which is
- * a probabilistic, 'maximum entropy' classifier.
- * <p>
- * Example:
- * <pre>
- * {@code
- * define TokenizeME datafu.pig.text.opennlp.TokenizeME('data/en-token.bin');
- *
- * -- input:
- * -- ("I believe the Masons have infiltrated the Apache PMC.")
- * infoo = LOAD 'input' AS (text:chararray);
-
- * -- output:
- * -- ({(I),(believe),(the),(Masons),(have),(infiltrated),(the),(Apache),(PMC),(.)})
- * outfoo = FOREACH input GENERATE TokenizeME(text) as tokens;
- * }
- * </pre>
- */
-
-
-
-public class TokenizeME extends EvalFunc<DataBag>
-{
- private TokenizerME tokenizer = null;
- private static final String MODEL_FILE = "tokens";
- private TupleFactory tf = TupleFactory.getInstance();
- private BagFactory bf = BagFactory.getInstance();
- private String modelPath;
-
- public TokenizeME(String modelPath) {
- this.modelPath = modelPath;
- }
-
- @Override
- public List<String> getCacheFiles() {
- List<String> list = new ArrayList<String>(1);
- list.add(this.modelPath + "#" + MODEL_FILE);
- return list;
- }
-
- // Enable multiple languages by specifying the model path. See http://text.sourceforge.net/models-1.5/
- public DataBag exec(Tuple input) throws IOException
- {
- if(input.size() != 1) {
- throw new IOException();
- }
-
- String inputString = input.get(0).toString();
- if(inputString == null || inputString == "") {
- return null;
- }
- DataBag outBag = bf.newDefaultBag();
- if(this.tokenizer == null) {
- String loadFile = CachedFile.getFileName(MODEL_FILE, this.modelPath);;
- InputStream file = new FileInputStream(loadFile);
- InputStream buffer = new BufferedInputStream(file);
- TokenizerModel model = new TokenizerModel(buffer);
- this.tokenizer = new TokenizerME(model);
- }
- String tokens[] = this.tokenizer.tokenize(inputString);
- for(String token : tokens) {
- Tuple outTuple = tf.newTuple(token);
- outBag.add(outTuple);
- }
- return outBag;
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try
- {
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.CHARARRAY)
- {
- throw new RuntimeException("Expected a CHARARRAY as input, but got a " + inputFieldSchema.toString());
- }
-
- Schema tupleSchema = new Schema();
- tupleSchema.add(new Schema.FieldSchema("token",DataType.CHARARRAY));
-
- return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
- .getName()
- .toLowerCase(), input),
- tupleSchema,
- DataType.BAG));
- }
- catch (FrontendException e)
- {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/text/opennlp/TokenizeSimple.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/text/opennlp/TokenizeSimple.java b/src/java/datafu/pig/text/opennlp/TokenizeSimple.java
deleted file mode 100644
index cea48b4..0000000
--- a/src/java/datafu/pig/text/opennlp/TokenizeSimple.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package datafu.pig.text.opennlp;
-
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
-import opennlp.tools.tokenize.SimpleTokenizer;
-import org.apache.pig.EvalFunc;
-import org.apache.pig.data.*;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * The OpenNLP Tokenizers segment an input character sequence into tokens. This one uses the OpenNLP class SimpleTokenizer
- * <p>
- * Example:
- * <pre>
- * {@code
- * define TokenizeSimple datafu.pig.text.opennlp.TokenizeSimple();
- *
- * -- input:
- * -- ("I believe the Masons have infiltrated the Apache PMC.")
- * infoo = LOAD 'input' AS (text:chararray);
-
- * -- output:
- * -- ({(I),(believe),(the),(Masons),(have),(infiltrated),(the),(Apache),(PMC),(.)})
- * outfoo = FOREACH input GENERATE TokenizeSimple(text) as tokens;
- * }
- * </pre>
- */
-public class TokenizeSimple extends EvalFunc<DataBag>
-{
- private SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
- private TupleFactory tf = TupleFactory.getInstance();
- private BagFactory bf = BagFactory.getInstance();
-
- public DataBag exec(Tuple input) throws IOException
- {
- if(input.size() != 1) {
- throw new IOException();
- }
-
- String inputString = input.get(0).toString();
- if(inputString == null || inputString == "") {
- return null;
- }
-
- DataBag outBag = bf.newDefaultBag();
- String tokens[] = tokenizer.tokenize(inputString);
- for(String token : tokens) {
- Tuple outTuple = tf.newTuple(token);
- outBag.add(outTuple);
- }
- return outBag;
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try
- {
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.CHARARRAY)
- {
- throw new RuntimeException("Expected a CHARARRAY as input, but got a " + inputFieldSchema.toString());
- }
-
- Schema tupleSchema = new Schema();
- tupleSchema.add(new Schema.FieldSchema("token",DataType.CHARARRAY));
-
- return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
- .getName()
- .toLowerCase(), input),
- tupleSchema,
- DataType.BAG));
- }
- catch (FrontendException e)
- {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/text/opennlp/TokenizeWhitespace.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/text/opennlp/TokenizeWhitespace.java b/src/java/datafu/pig/text/opennlp/TokenizeWhitespace.java
deleted file mode 100644
index 8efafb0..0000000
--- a/src/java/datafu/pig/text/opennlp/TokenizeWhitespace.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package datafu.pig.text.opennlp;
-
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
-import opennlp.tools.tokenize.WhitespaceTokenizer;
-import org.apache.pig.EvalFunc;
-import org.apache.pig.data.*;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * The OpenNLP Tokenizers segment an input character sequence into tokens. This one uses the OpenNLP class
- * WhitespaceTokenizer.
- * <p>
- * Example:
- * <pre>
- * {@code
- * define TokenizeWhitespace datafu.pig.text.opennlp.TokenizeWhitespace();
- *
- * -- input:
- * -- ("I believe the Masons have infiltrated the Apache PMC.")
- * infoo = LOAD 'input' AS (text:chararray);
-
- * -- output:
- * -- ({(I),(believe),(the),(Masons),(have),(infiltrated),(the),(Apache),(PMC),(.)})
- * outfoo = FOREACH input GENERATE TokenizeWhitespace(text) as tokens;
- * }
- * </pre>
- */
-public class TokenizeWhitespace extends EvalFunc<DataBag>
-{
- private WhitespaceTokenizer tokenizer = WhitespaceTokenizer.INSTANCE;
- private TupleFactory tf = TupleFactory.getInstance();
- private BagFactory bf = BagFactory.getInstance();
-
- public DataBag exec(Tuple input) throws IOException
- {
- if(input.size() != 1) {
- throw new IOException();
- }
-
- String inputString = input.get(0).toString();
- if(inputString == null || inputString == "") {
- return null;
- }
-
- DataBag outBag = bf.newDefaultBag();
- String tokens[] = tokenizer.tokenize(inputString);
- for(String token : tokens) {
- Tuple outTuple = tf.newTuple(token);
- outBag.add(outTuple);
- }
- return outBag;
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try
- {
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.CHARARRAY)
- {
- throw new RuntimeException("Expected a CHARARRAY as input, but got a " + inputFieldSchema.toString());
- }
-
- Schema tupleSchema = new Schema();
- tupleSchema.add(new Schema.FieldSchema("token",DataType.CHARARRAY));
-
- return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
- .getName()
- .toLowerCase(), input),
- tupleSchema,
- DataType.BAG));
- }
- catch (FrontendException e)
- {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/urls/UserAgentClassify.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/urls/UserAgentClassify.java b/src/java/datafu/pig/urls/UserAgentClassify.java
deleted file mode 100644
index 4040fea..0000000
--- a/src/java/datafu/pig/urls/UserAgentClassify.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.urls;
-
-import datafu.pig.util.SimpleEvalFunc;
-
-/**
- * Given a user agent string, this UDF classifies clients to 'mobile' and 'desktop'.
- * Current as of June 2011.
- */
-public class UserAgentClassify extends SimpleEvalFunc<String>
-{
-
- public String call(String useragent)
- {
- if (useragent.length() < 4)
- return "desktop"; //
- String ua=useragent.toLowerCase();
- if(ua.matches(".*(android|avantgo|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\\/|plucker|pocket|psp|symbian|treo|up\\.(browser|link)|vodafone|wap|windows (ce|phone)|xda|xiino).*")||ua.substring(0,4).matches("1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s\\-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|\\-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw\\-(n|u)|c55\\/|capi|ccwa|cdm\\-|cell|chtm|cldc|cmd\\-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc\\-s|devi|dica|dmob|do(c|p)o|ds(12|\\-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(\\-|_)|g1 u|g560|gene|gf\\-5|g\\-mo|go(\\.w|od)|gr(ad|un)|haie|hcit|hd\\-(m|p|t)|hei\\-|hi(pt|ta)|hp( i|ip)|hs\\-c|ht(c(\\-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i\\-(20|go|ma)|i230|iac( |\\-|\\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\\/)|klon|kp
t |kwc\\-|kyo(c|k)|le(no|xi)|lg( g|\\/(k|l|u)|50|54|e\\-|e\\/|\\-[a-w])|libw|lynx|m1\\-w|m3ga|m50\\/|ma(te|ui|xo)|mc(01|21|ca)|m\\-cr|me(di|rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(\\-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)\\-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|\\-([1-8]|c))|phil|pire|pl(ay|uc)|pn\\-2|po(ck|rt|se)|prox|psio|pt\\-g|qa\\-a|qc(07|12|21|32|60|\\-[2-7]|i\\-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h\\-|oo|p\\-)|sdk\\/|se(c(\\-|0|1)|47|mc|nd|ri)|sgh\\-|shar|sie(\\-|m)|sk\\-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h\\-|v\\-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl\\-|tdg\\-|tel(i|m)|tim\\-|t\\-mo|to(pl|sh)|ts(70|m\\-|m3|m5)|tx\\-9|up(\\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|\\-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(\\-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|xda(\\-|2|g)|yas\\-|your|zeto|zte
\\-"))
- return "mobile";
- else
- return "desktop";
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/urls/package-info.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/urls/package-info.java b/src/java/datafu/pig/urls/package-info.java
deleted file mode 100644
index 332beda..0000000
--- a/src/java/datafu/pig/urls/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * UDFs for processing URLs.
- */
-package datafu.pig.urls;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/util/AliasableEvalFunc.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/util/AliasableEvalFunc.java b/src/java/datafu/pig/util/AliasableEvalFunc.java
deleted file mode 100644
index ee2c3f3..0000000
--- a/src/java/datafu/pig/util/AliasableEvalFunc.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.util;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * Makes implementing and using UDFs easier by enabling named parameters.
- *
- * <p>
- * This works by capturing the schema of the input tuple on the front-end and storing it into the UDFContext.
- * It provides an easy means of referencing the parameters on the back-end to aid in writing schema-based UDFs.
- * </p>
- *
- * <p>
- * A related class is {@link SimpleEvalFunc}. However they are actually fairly different. The primary purpose of {@link SimpleEvalFunc} is
- * to skip the boilerplate under the assumption that the arguments in and out are well... simple.
- * It also assumes that these arguments are in a well-defined positional ordering.
- * </p>
- *
- * <p>
- * AliasableEvalFunc allows the UDF writer to avoid dealing with all positional assumptions and instead reference fields
- * by their aliases. This practice allows for more readable code since the alias names should have more meaning
- * to the reader than the position. This approach is also less error prone since it creates a more explicit contract
- * for what input the UDF expects and prevents simple mistakes that positional-based UDFs could not easily catch,
- * such as transposing two fields of the same type. If this contract is violated, say, by attempting to reference
- * a field that is not present, a meaningful error message may be thrown.
- * </p>
- *
- * <p>
- * Example: This example computes the monthly payments for mortgages depending on interest rate.
- * <pre>
- * {@code
- * public class MortgagePayment extends AliasableEvalFunc<DataBag> {
- * ...
- * public DataBag exec(Tuple input) throws IOException {
- * DataBag output = BagFactory.getInstance().newDefaultBag();
- *
- * Double principal = getDouble(input, "principal"); // get a value from the input tuple by alias
- * Integer numPayments = getInteger(input, "num_payments");
- * DataBag interestRates = getBag(input, "interest_rates");
- *
- * for (Tuple interestTuple : interestRates) {
- * Double interest = getDouble(interestTuple, getPrefixedAliasName("interest_rates", "interest_rate")); // get a value from the inner bag tuple by alias
- * double monthlyPayment = computeMonthlyPayment(principal, numPayments, interest);
- * output.add(TupleFactory.getInstance().newTuple(monthlyPayment));
- * }
- * return output;
- * }
- * }
- * }
- * </pre>
- * </p>
- *
- * @author wvaughan
- *
- * @param <T>
- */
-public abstract class AliasableEvalFunc<T> extends ContextualEvalFunc<T>
-{
- private static final String ALIAS_MAP_PROPERTY = "aliasMap";
-
- private Map<String, Integer> aliasToPosition = null;
-
- public AliasableEvalFunc() {
-
- }
-
- /**
- * A wrapper method which captures the schema and then calls getOutputSchema
- */
- @Override
- public Schema outputSchema(Schema input) {
- storeFieldAliases(input);
- return getOutputSchema(input);
- }
-
- /**
- * Specify the output schema as in {link EvalFunc#outputSchema(Schema)}.
- *
- * @param input
- * @return outputSchema
- */
- public abstract Schema getOutputSchema(Schema input);
-
- @SuppressWarnings("unchecked")
- private Map<String, Integer> getAliasMap() {
- return (Map<String, Integer>)getInstanceProperties().get(ALIAS_MAP_PROPERTY);
- }
-
- private void setAliasMap(Map<String, Integer> aliases) {
- getInstanceProperties().put(ALIAS_MAP_PROPERTY, aliases);
- }
-
- private void storeFieldAliases(Schema tupleSchema)
- {
- Map<String, Integer> aliases = new HashMap<String, Integer>();
- constructFieldAliases(aliases, tupleSchema, null);
- log.debug("In instance: "+getInstanceName()+", stored alias map: " + aliases);
-
- // pass the input schema into the exec function
- setAliasMap(aliases);
- }
-
- private void constructFieldAliases(Map<String, Integer> aliases, Schema tupleSchema, String prefix)
- {
- int position = 0;
- for (Schema.FieldSchema field : tupleSchema.getFields()) {
- String alias = getPrefixedAliasName(prefix, field.alias);
- if (field.alias != null && !field.alias.equals("null")) {
- aliases.put(alias, position);
- log.debug("In instance: "+getInstanceName()+", stored alias " + alias + " as position " + position);
- }
- if (field.schema != null) {
- constructFieldAliases(aliases, field.schema, alias);
- }
- position++;
- }
- }
-
- public String getPrefixedAliasName(String prefix, String alias)
- {
- if (alias == null || alias.equals("null")) {
- if (prefix == null) return "";
- else return prefix; // ignore the null inner bags/tuples
- }
- else return ((prefix == null || prefix.equals("null") || prefix.trim().equals("")) ? "" : prefix+".") + alias; // handle top bag
- }
-
- /**
- * Field aliases are generated from the input schema<br/>
- * Each alias maps to a bag position<br/>
- * Inner bags/tuples will have alias of outer.inner.foo
- *
- * @return A map of field alias to field position
- */
- public Map<String, Integer> getFieldAliases()
- {
- Map<String, Integer> aliases = getAliasMap();
- if (aliases == null) {
- log.error("Class: " + this.getClass());
- log.error("Instance name: " + this.getInstanceName());
- log.error("Properties: " + getContextProperties());
- throw new RuntimeException("Could not retrieve aliases from properties using " + ALIAS_MAP_PROPERTY);
- }
- return aliases;
- }
-
- public Integer getPosition(String alias) {
- if (aliasToPosition == null) {
- aliasToPosition = getFieldAliases();
- }
- return aliasToPosition.get(alias);
- }
-
- public Integer getPosition(String prefix, String alias) {
- return getPosition(getPrefixedAliasName(prefix, alias));
- }
-
- public Integer getInteger(Tuple tuple, String alias) throws ExecException {
- return getInteger(tuple, alias, null);
- }
-
- public Integer getInteger(Tuple tuple, String alias, Integer defaultValue) throws ExecException {
- Integer i = getPosition(alias);
- if (i == null) throw new FieldNotFound("Attempt to reference unknown alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
- if (i >= tuple.size()) throw new FieldNotFound("Attempt to reference outside of tuple for alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
- Number number = (Number)tuple.get(i);
- if (number == null) return defaultValue;
- return number.intValue();
- }
-
- public Long getLong(Tuple tuple, String alias) throws ExecException {
- return getLong(tuple, alias, null);
- }
-
- public Long getLong(Tuple tuple, String alias, Long defaultValue) throws ExecException {
- Integer i = getPosition(alias);
- if (i == null) throw new FieldNotFound("Attempt to reference unknown alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
- if (i >= tuple.size()) throw new FieldNotFound("Attempt to reference outside of tuple for alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
- Number number = (Number)tuple.get(i);
- if (number == null) return defaultValue;
- return number.longValue();
- }
-
- public Float getFloat(Tuple tuple, String alias) throws ExecException {
- return getFloat(tuple, alias, null);
- }
-
- public Float getFloat(Tuple tuple, String alias, Float defaultValue) throws ExecException {
- Integer i = getPosition(alias);
- if (i == null) throw new FieldNotFound("Attempt to reference unknown alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
- if (i >= tuple.size()) throw new FieldNotFound("Attempt to reference outside of tuple for alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
- Number number = (Number)tuple.get(i);
- if (number == null) return defaultValue;
- return number.floatValue();
- }
-
- public Double getDouble(Tuple tuple, String alias) throws ExecException {
- return getDouble(tuple, alias, null);
- }
-
- public Double getDouble(Tuple tuple, String alias, Double defaultValue) throws ExecException {
- Integer i = getPosition(alias);
- if (i == null) throw new FieldNotFound("Attempt to reference unknown alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
- if (i >= tuple.size()) throw new FieldNotFound("Attempt to reference outside of tuple for alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
- Number number = (Number)tuple.get(i);
- if (number == null) return defaultValue;
- return number.doubleValue();
- }
-
- public String getString(Tuple tuple, String alias) throws ExecException {
- return getString(tuple, alias, null);
- }
-
- public String getString(Tuple tuple, String alias, String defaultValue) throws ExecException {
- Integer i = getPosition(alias);
- if (i == null) throw new FieldNotFound("Attempt to reference unknown alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
- if (i >= tuple.size()) throw new FieldNotFound("Attempt to reference outside of tuple for alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
- String s = (String)tuple.get(i);
- if (s == null) return defaultValue;
- return s;
- }
-
- public Boolean getBoolean(Tuple tuple, String alias) throws ExecException {
- Integer i = getPosition(alias);
- if (i == null) throw new FieldNotFound("Attempt to reference unknown alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
- if (i >= tuple.size()) throw new FieldNotFound("Attempt to reference outside of tuple for alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
- return (Boolean)tuple.get(i);
- }
-
- public DataBag getBag(Tuple tuple, String alias) throws ExecException {
- Integer i = getPosition(alias);
- if (i == null) throw new FieldNotFound("Attempt to reference unknown alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
- if (i >= tuple.size()) throw new FieldNotFound("Attempt to reference outside of tuple for alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
- return (DataBag)tuple.get(i);
- }
-
- public Object getObject(Tuple tuple, String alias) throws ExecException {
- Integer i = getPosition(alias);
- if (i == null) throw new FieldNotFound("Attempt to reference unknown alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
- if (i >= tuple.size()) throw new FieldNotFound("Attempt to reference outside of tuple for alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
- return tuple.get(i);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/util/Assert.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/util/Assert.java b/src/java/datafu/pig/util/Assert.java
deleted file mode 100644
index a258915..0000000
--- a/src/java/datafu/pig/util/Assert.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.util;
-
-/**
- * Assert has been renamed to AssertUDF.
- *
- * This class is provided for backward compatibility.
- *
- * @deprecated Use {@link AssertUDF} instead.
- */
- @Deprecated
-public class Assert extends AssertUDF
-{
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/util/AssertUDF.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/util/AssertUDF.java b/src/java/datafu/pig/util/AssertUDF.java
deleted file mode 100644
index 16f9247..0000000
--- a/src/java/datafu/pig/util/AssertUDF.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.util;
-
-import java.io.IOException;
-
-import org.apache.pig.FilterFunc;
-import org.apache.pig.data.Tuple;
-
-/**
- * Filter function which asserts that a value is true.
- *
- * <p>
- * Unfortunately, the Pig interpreter doesn't recognize boolean expressions nested as function
- * arguments, so this uses C-style booleans. That is, the first argument should be
- * an integer. 0 is interpreted as "false", and anything else is considered "true".
- * The function will cause the Pig script to fail if a "false" value is encountered.
- * </p>
- *
- * <p>
- * There is a unary and a binary version. The unary version just takes a boolean, and throws out a generic exception message when the
- * assertion is violated. The binary version takes a String as a second argument and throws that out when the assertion
- * is violated.
- * </p>
- *
- * <p>
- * Example:
- * <pre>
- * {@code
- * FILTER members BY AssertUDF( (member_id >= 0 ? 1 : 0), 'Doh! Some member ID is negative.' );
- * }
- * </pre>
- * </p>
- */
-public class AssertUDF extends FilterFunc
-{
- @Override
- public Boolean exec(Tuple tuple)
- throws IOException
- {
- if ((Integer) tuple.get(0) == 0) {
- if (tuple.size() > 1) {
- throw new IOException("Assertion violated: " + tuple.get(1).toString());
- }
- else {
- throw new IOException("Assertion violated. What assertion, I do not know, but it was officially violated.");
- }
- }
- else {
- return true;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/util/BoolToInt.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/util/BoolToInt.java b/src/java/datafu/pig/util/BoolToInt.java
deleted file mode 100644
index 5ec8d3a..0000000
--- a/src/java/datafu/pig/util/BoolToInt.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.util;
-
-/**
- * UDF which converts a Boolean to an Integer.
- */
-public class BoolToInt extends SimpleEvalFunc<Integer>
-{
- public Integer call(Boolean val)
- {
- return (val == null || !val) ? 0 : 1;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/util/Coalesce.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/util/Coalesce.java b/src/java/datafu/pig/util/Coalesce.java
deleted file mode 100644
index f8e25f4..0000000
--- a/src/java/datafu/pig/util/Coalesce.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.util;
-
-import java.io.IOException;
-
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
-
-/**
- * Returns the first non-null value from a tuple, just like {@link <a href="http://msdn.microsoft.com/en-us/library/ms190349.aspx" target="_blank">COALESCE</a>} in SQL.
- *
- * <p>
- * Example:
- * <pre>
- * {@code
- *
- * define COALESCE datafu.pig.util.COALESCE();
-
- * -- input: 1,2,3,NULL,4,NULL,5
- * input = LOAD 'input' AS (val:int);
- *
- * -- produces: 1,2,3,99,4,99,5
- * coalesced = FOREACH input GENERATE COALESCE(val,99);
- *
- * }
- * </pre>
- * </p>
- *
- * @author "Matthew Hayes <mh...@linkedin.com>"
- *
- */
-public class Coalesce extends AliasableEvalFunc<Object>
-{
- private boolean strict;
-
- private static String STRICT_OPTION = "strict";
- private static String LAZY_OPTION = "lazy";
-
- public Coalesce()
- {
- strict = true;
- }
-
- public Coalesce(String option)
- {
- if (option.equals(STRICT_OPTION))
- {
- strict = true;
- }
- else if (option.equals(LAZY_OPTION))
- {
- strict = false;
- }
- else
- {
- throw new IllegalArgumentException("Unexpected option: " + option);
- }
- }
-
- @Override
- public Object exec(Tuple input) throws IOException
- {
- if (input == null || input.size() == 0)
- {
- return null;
- }
-
- Byte type = (Byte)getInstanceProperties().get("type");
-
- for (Object o : input)
- {
- if (o != null)
- {
- if (strict)
- {
- return o;
- }
- else
- {
- try
- {
- switch (type)
- {
- case DataType.INTEGER:
- return DataType.toInteger(o);
- case DataType.LONG:
- return DataType.toLong(o);
- case DataType.DOUBLE:
- return DataType.toDouble(o);
- case DataType.FLOAT:
- return DataType.toFloat(o);
- default:
- return o;
- }
- }
- catch (Exception e)
- {
- DataFuException dfe = new DataFuException(e.getMessage(),e);
- dfe.setData(o);
- dfe.setFieldAliases(getFieldAliases());
- throw dfe;
- }
- }
- }
- }
-
- return null;
- }
-
- @Override
- public Schema getOutputSchema(Schema input)
- {
- if (input.getFields().size() == 0)
- {
- throw new RuntimeException("Expected at least one parameter");
- }
-
- Byte outputType = null;
- int pos = 0;
- for (FieldSchema field : input.getFields())
- {
- if (DataType.isSchemaType(field.type))
- {
- throw new RuntimeException(String.format("Not supported on schema types. Found %s in position %d.",DataType.findTypeName(field.type),pos));
- }
-
- if (DataType.isComplex(field.type))
- {
- throw new RuntimeException(String.format("Not supported on complex types. Found %s in position %d.",DataType.findTypeName(field.type),pos));
- }
-
- if (!DataType.isUsableType(field.type))
- {
- throw new RuntimeException(String.format("Not a usable type. Found %s in position %d.",DataType.findTypeName(field.type),pos));
- }
-
- if (outputType == null)
- {
- outputType = field.type;
- }
- else if (!outputType.equals(field.type))
- {
- if (strict)
- {
- throw new RuntimeException(String.format("Expected all types to be equal, but found '%s' in position %d. First element has type '%s'. "
- + "If you'd like to attempt merging types, use the '%s' option, as '%s' is the default.",
- DataType.findTypeName(field.type),pos,DataType.findTypeName((byte)outputType),LAZY_OPTION,STRICT_OPTION));
- }
- else
- {
- byte merged = DataType.mergeType(outputType, field.type);
- if (merged == DataType.ERROR)
- {
- throw new RuntimeException(String.format("Expected all types to be equal, but found '%s' in position %d, where output type is '%s', and types could not be merged.",
- DataType.findTypeName(field.type),pos,DataType.findTypeName((byte)outputType)));
- }
- outputType = merged;
- }
- }
-
- pos++;
- }
-
- getInstanceProperties().put("type", outputType);
-
- return new Schema(new Schema.FieldSchema("item",outputType));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/util/ContextualEvalFunc.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/util/ContextualEvalFunc.java b/src/java/datafu/pig/util/ContextualEvalFunc.java
deleted file mode 100644
index c534b77..0000000
--- a/src/java/datafu/pig/util/ContextualEvalFunc.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.util;
-
-import java.util.Properties;
-
-import org.apache.pig.EvalFunc;
-import org.apache.pig.impl.util.UDFContext;
-
-/**
- * An abstract class which enables UDFs to store instance properties
- * on the front end which will be available on the back end.
- * For example, properties may be set in the call to outputSchema(),
- * which will be available when exec() is called.
- *
- * @param <T>
- */
-public abstract class ContextualEvalFunc<T> extends EvalFunc<T>
-{
- private String instanceName;
-
- @Override
- public void setUDFContextSignature(String signature) {
- setInstanceName(signature);
- }
-
- /**
- * Helper method to return the context properties for this class
- *
- * @return context properties
- */
- protected Properties getContextProperties() {
- UDFContext context = UDFContext.getUDFContext();
- Properties properties = context.getUDFProperties(this.getClass());
- return properties;
- }
-
- /**
- * Helper method to return the context properties for this instance of this class
- *
- * @return instances properties
- */
- protected Properties getInstanceProperties() {
- Properties contextProperties = getContextProperties();
- if (!contextProperties.containsKey(getInstanceName())) {
- contextProperties.put(getInstanceName(), new Properties());
- }
- return (Properties)contextProperties.get(getInstanceName());
- }
-
- /**
- *
- * @return the name of this instance corresponding to the UDF Context Signature
- * @see #setUDFContextSignature(String)
- */
- protected String getInstanceName() {
- if (instanceName == null) {
- throw new RuntimeException("Instance name is null. This should not happen unless UDFContextSignature was not set.");
- }
- return instanceName;
- }
-
- private void setInstanceName(String instanceName) {
- this.instanceName = instanceName;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/util/DataFuException.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/util/DataFuException.java b/src/java/datafu/pig/util/DataFuException.java
deleted file mode 100644
index 0066aa8..0000000
--- a/src/java/datafu/pig/util/DataFuException.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.util;
-
-import java.util.Map;
-
-public class DataFuException extends RuntimeException
-{
- private static final long serialVersionUID = 1L;
- private Map<String, Integer> fieldAliases;
- private Object data;
-
- public DataFuException()
- {
- super();
- }
-
- public DataFuException(String message)
- {
- super(message);
- }
-
- public DataFuException(String message, Throwable cause)
- {
- super(message, cause);
- }
-
- public DataFuException(Throwable cause)
- {
- super(cause);
- }
-
- /**
- * Gets field aliases for a UDF which may be relevant to this exception.
- *
- * @return field aliases
- */
- public Map<String, Integer> getFieldAliases()
- {
- return fieldAliases;
- }
-
- /**
- * Gets data relevant to this exception.
- *
- * @return data
- */
- public Object getData()
- {
- return data;
- }
-
- /**
- * Sets field aliases for a UDF which may be relevant to this exception.
- *
- * @param fieldAliases
- */
- public void setFieldAliases(Map<String, Integer> fieldAliases)
- {
- this.fieldAliases = fieldAliases;
- }
-
- /**
- * Sets data relevant to this exception.
- * @param data
- */
- public void setData(Object data)
- {
- this.data = data;
- }
-
- @Override
- public String toString()
- {
- String s = getClass().getName();
- String message = getLocalizedMessage();
-
- StringBuilder result = new StringBuilder(s);
-
- if (message != null)
- {
- result.append(": ");
- result.append(message);
- }
-
- if (getFieldAliases() != null)
- {
- result.append("\nAliases:");
- for (String alias : getFieldAliases().keySet())
- {
- result.append("\n");
- result.append(alias != null && alias.length() > 0 ? alias : "???");
- }
- }
-
- if (getData() != null)
- {
- result.append("\nData:");
- result.append("\n");
- result.append(data.toString());
- }
-
- return result.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/util/FieldNotFound.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/util/FieldNotFound.java b/src/java/datafu/pig/util/FieldNotFound.java
deleted file mode 100644
index d624007..0000000
--- a/src/java/datafu/pig/util/FieldNotFound.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.util;
-
-import org.apache.pig.backend.executionengine.ExecException;
-
-/**
- * Thrown by {see AliasableEvalFunc} when attempting to access an unknown field by name.
- *
- * @author wvaughan
- *
- */
-public class FieldNotFound extends ExecException
-{
- private static final long serialVersionUID = 1L;
-
- public FieldNotFound() {
- super();
- }
-
- public FieldNotFound(String message) {
- super(message);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/util/In.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/util/In.java b/src/java/datafu/pig/util/In.java
deleted file mode 100644
index 0667914..0000000
--- a/src/java/datafu/pig/util/In.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.util;
-
-/**
- * In has been renamed to InUDF.
- *
- * This class is provided for backward compatibility.
- *
- * @deprecated Use {@link InUDF} instead.
- */
- @Deprecated
-public class In extends InUDF
-{
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/util/InUDF.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/util/InUDF.java b/src/java/datafu/pig/util/InUDF.java
deleted file mode 100644
index 5057285..0000000
--- a/src/java/datafu/pig/util/InUDF.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.util;
-
-import java.io.IOException;
-
-import org.apache.pig.FilterFunc;
-import org.apache.pig.data.Tuple;
-
-/**
- * Similar to the SQL IN function, this function provides a convenient way to filter
- * using a logical disjunction over many values.
- * Returns true when the first value of the tuple is contained within the remainder of the tuple.
- *
- * <p>
- * Example:
- * <pre>
- * {@code
- * define In datafu.pig.util.InUDF();
- * -- cars: (alice, red), (bob, blue), (charlie, green), (dave, red);
- * cars = LOAD cars AS (owner:chararray, color:chararray);
- *
- * -- cars: (alice, red), (bob, blue), (dave, red);
- * red_blue_cars = FILTER cars BY In(color, 'red', 'blue');
- *
- * }</pre>
- * </p>
- *
- * @author wvaughan
- *
- */
-public class InUDF extends FilterFunc
-{
-
- @Override
- public Boolean exec(Tuple input) throws IOException
- {
- Object o = input.get(0);
- Boolean match = false;
- if (o != null) {
- for (int i=1; i<input.size() && !match; i++) {
- match = match || o.equals(input.get(i));
- }
- }
- return match;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/util/IntToBool.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/util/IntToBool.java b/src/java/datafu/pig/util/IntToBool.java
deleted file mode 100644
index d00e297..0000000
--- a/src/java/datafu/pig/util/IntToBool.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.util;
-
-/**
- * UDF which converts an Integer to a Boolean.
- */
-public class IntToBool extends SimpleEvalFunc<Boolean>
-{
- public Boolean call(Integer val)
- {
- return (val == null || val == 0) ? false : true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/util/SimpleEvalFunc.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/util/SimpleEvalFunc.java b/src/java/datafu/pig/util/SimpleEvalFunc.java
deleted file mode 100644
index 2d262b4..0000000
--- a/src/java/datafu/pig/util/SimpleEvalFunc.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.util;
-
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.lang.reflect.Type;
-
-import org.apache.pig.EvalFunc;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-
-/**
- Uses reflection to makes writing simple wrapper Pig UDFs easier.
-
- For example, writing a simple string trimming UDF might look like
- this:
- <pre>
- {@code
- public class TRIM extends EvalFunc<String>
- {
- public String exec(Tuple input) throws IOException
- {
- if (input.size() != 1)
- throw new IllegalArgumentException("requires a parameter");
-
- try {
- Object o = input.get(0);
- if (!(o instanceof String))
- throw new IllegalArgumentException("expected a string");
-
- String str = (String)o;
- return (str == null) ? null : str.trim();
- }
- catch (Exception e) {
- throw WrappedIOException.wrap("error...", e);
- }
- }
- }
- }
- </pre>
- There is a lot of boilerplate to check the number of arguments and
- the parameter types in the tuple.
-
- Instead, with this class, you can derive from SimpleEvalFunc and
- create a <code>call()</code> method (not exec!), just specifying the
- arguments as a regular function. The class handles all the argument
- checking and exception wrapping for you. So your code would be:
- <pre>
- {@code
- public class TRIM2 extends SimpleEvalFunc<String>
- {
- public String call(String s)
- {
- return (s != null) ? s.trim() : null;
- }
- }
- }
- </pre>
-
- An example of this UDF in action with Pig:
- <pre>
- {@code
- grunt> a = load 'test' as (x:chararray, y:chararray); dump a;
- (1 , 2)
-
- grunt> b = foreach a generate TRIM2(x); dump b;
- (1)
-
- grunt> c = foreach a generate TRIM2((int)x); dump c;
- datafu.pig.util.TRIM2(java.lang.String): argument type
- mismatch [#1]; expected java.lang.String, got java.lang.Integer
-
- grunt> d = foreach a generate TRIM2(x, y); dump d;
- datafu.pig.util.TRIM2(java.lang.String): got 2 arguments, expected 1.
- }
- </pre>
-
-*/
-
-public abstract class SimpleEvalFunc<T> extends EvalFunc<T>
-{
- // TODO Add support for other UDF types (e.g., FilterFunc)
- // TODO Algebraic EvalFuncs
-
- Method m = null;
-
- public SimpleEvalFunc()
- {
- for (Method method : this.getClass().getMethods()) {
- if (method.getName() == "call")
- m = method;
- }
- if (m == null)
- throw new IllegalArgumentException(String.format("%s: couldn't find call() method in UDF.", getClass().getName()));
- }
-
- // Pig can't get the return type via reflection (as getReturnType normally tries to do), so give it a hand
- @Override
- public Type getReturnType()
- {
- return m.getReturnType();
- }
-
- private String _method_signature()
- {
- StringBuilder sb = new StringBuilder(getClass().getName());
- Class<?> pvec[] = m.getParameterTypes();
-
- sb.append("(");
- for (int i=0; i < pvec.length; i++) {
- if (i > 0)
- sb.append(", ");
- sb.append(String.format("%s", pvec[i].getName()));
- }
- sb.append(")");
-
- return sb.toString();
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public T exec(Tuple input) throws IOException
- {
- @SuppressWarnings("rawtypes")
- Class pvec[] = m.getParameterTypes();
-
- if (input == null || input.size() == 0)
- return null;
-
- // check right number of arguments
- if (input.size() != pvec.length)
- throw new IOException(String.format("%s: got %d arguments, expected %d.", _method_signature(), input.size(), pvec.length));
-
- // pull and check argument types
- Object[] args = new Object[input.size()];
- for (int i=0; i < pvec.length; i++) {
- Object o = input.get(i);
- try {
- o = pvec[i].cast(o);
- }
- catch (ClassCastException e) {
- throw new IOException(String.format("%s: argument type mismatch [#%d]; expected %s, got %s", _method_signature(), i+1,
- pvec[i].getName(), o.getClass().getName()));
- }
- args[i] = o;
- }
-
- try {
- return (T) m.invoke(this, args);
- }
- catch (Exception e) {
- throw new IOException(String.format("%s: caught exception processing input.", _method_signature()), e);
- }
- }
-
- /**
- * Override outputSchema so we can verify the input schema at pig compile time, instead of runtime
- * @param inputSchema input schema
- * @return call to super.outputSchema in case schema was defined elsewhere
- */
- @Override
- public Schema outputSchema(Schema inputSchema)
- {
- if (inputSchema == null) {
- throw new IllegalArgumentException(String.format("%s: null schema passed to %s", _method_signature(), getClass().getName()));
- }
-
- // check correct number of arguments
- @SuppressWarnings("rawtypes")
- Class parameterTypes[] = m.getParameterTypes();
- if (inputSchema.size() != parameterTypes.length) {
- throw new IllegalArgumentException(String.format("%s: got %d arguments, expected %d.",
- _method_signature(),
- inputSchema.size(),
- parameterTypes.length));
- }
-
- // check type for each argument
- for (int i=0; i < parameterTypes.length; i++) {
- try {
- byte inputType = inputSchema.getField(i).type;
- byte parameterType = DataType.findType(parameterTypes[i]);
- if (inputType != parameterType) {
- throw new IllegalArgumentException(String.format("%s: argument type mismatch [#%d]; expected %s, got %s",
- _method_signature(),
- i+1,
- DataType.findTypeName(parameterType),
- DataType.findTypeName(inputType)));
- }
- }
- catch (FrontendException fe) {
- throw new IllegalArgumentException(String.format("%s: Problem with input schema: ", _method_signature(), inputSchema), fe);
- }
- }
-
- // delegate to super to determine the actual outputSchema (if specified)
- return super.outputSchema(inputSchema);
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/util/TransposeTupleToBag.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/util/TransposeTupleToBag.java b/src/java/datafu/pig/util/TransposeTupleToBag.java
deleted file mode 100644
index f8a39df..0000000
--- a/src/java/datafu/pig/util/TransposeTupleToBag.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.util;
-
-import java.io.IOException;
-import java.util.HashMap;
-
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
-
-/**
- * Performs a transpose on a tuple, resulting in a bag of key, value fields where
- * the key is the column name and the value is the value of that column in the tuple.
- *
- * <p>
- * Example:
- * <pre>
- * {@code
- *
- * define TransposeTupleToBag datafu.pig.util.TransposeTupleToBag();
-
- * -- input: 1,10,11,12
- * input = LOAD 'input' AS (id:int,val1:int,val2:int,val3:int);
- *
- * -- produces: 1,{("val1",10),("val2",11),("val3",12)}
- * output = FOREACH input GENERATE id, TransposeTupleToBag(val1 .. val3);
- *
- * }
- * </pre>
- * </p>
- *
- * @author "William Vaughan <wv...@linkedin.com>"
- *
- */
-public class TransposeTupleToBag extends AliasableEvalFunc<DataBag>
-{
- private final String TRANSPOSE_TYPE = "TRANSPOSE_TYPE";
-
- @Override
- public Schema getOutputSchema(Schema input)
- {
- try
- {
- // require that every field in the input has the same type
- Byte type = null;
- for (FieldSchema fieldSchema : input.getFields()) {
- if (type == null) {
- type = fieldSchema.type;
- } else {
- if (type != fieldSchema.type) {
- throw new RuntimeException(
- String.format("Expected all input types to match. Got both %s and %s.",
- DataType.findTypeName(type.byteValue()), DataType.findTypeName(fieldSchema.type)));
- }
- }
- }
- getInstanceProperties().put(TRANSPOSE_TYPE, type);
-
- Schema outputTupleSchema = new Schema();
- outputTupleSchema.add(new Schema.FieldSchema("key", DataType.CHARARRAY));
- outputTupleSchema.add(new Schema.FieldSchema("value", type));
- return new Schema(new Schema.FieldSchema(
- getSchemaName(this.getClass().getName().toLowerCase(), input),
- outputTupleSchema,
- DataType.BAG));
- }
- catch (FrontendException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public DataBag exec(Tuple input) throws IOException
- {
- // initialize a reverse mapping
- HashMap<Integer, String> positionToAlias = new HashMap<Integer, String>();
- for (String alias : getFieldAliases().keySet()) {
- positionToAlias.put(getFieldAliases().get(alias), alias);
- }
- DataBag output = BagFactory.getInstance().newDefaultBag();
- for (int i=0; i<input.size(); i++) {
- Tuple tuple = TupleFactory.getInstance().newTuple();
- tuple.append(positionToAlias.get(i));
- tuple.append(input.get(i));
- output.add(tuple);
- }
- return output;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/util/package-info.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/util/package-info.java b/src/java/datafu/pig/util/package-info.java
deleted file mode 100644
index cce46a0..0000000
--- a/src/java/datafu/pig/util/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Other useful utilities.
- */
-package datafu.pig.util;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test.sh
----------------------------------------------------------------------
diff --git a/test.sh b/test.sh
index f017e58..f4b479b 100755
--- a/test.sh
+++ b/test.sh
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
echo $$ > test.pid
-ant test
+gradle test
rm test.pid
[12/19] DATAFU-27 Migrate build system to Gradle
Posted by wv...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/sampling/SimpleRandomSampleTestOld.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/sampling/SimpleRandomSampleTestOld.java b/datafu-pig/src/test/java/datafu/test/pig/sampling/SimpleRandomSampleTestOld.java
new file mode 100644
index 0000000..20f5984
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/sampling/SimpleRandomSampleTestOld.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.sampling;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.pig.sampling.SimpleRandomSample;
+import datafu.test.pig.PigTests;
+
+/**
+ * Tests for {@link SimpleRandomSample}.
+ *
+ * @deprecated This tests the deprecated functionality of SimpleRandomSample
+ * where the probability can be specified in the constructor.
+ * @author ximeng
+ *
+ */
+public class SimpleRandomSampleTestOld extends PigTests
+{
+ /**
+ *
+ *
+ * DEFINE SRS datafu.pig.sampling.SimpleRandomSample('$SAMPLING_PROBABILITY');
+ *
+ * data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
+ *
+ * sampled = FOREACH (GROUP data ALL) GENERATE SRS(data) as sample_data;
+ *
+ * sampled = FOREACH sampled GENERATE COUNT(sample_data) AS sample_count;
+ *
+ * STORE sampled INTO 'output';
+ */
+ @Multiline
+ private String simpleRandomSampleTest;
+
+ @Test
+ public void simpleRandomSampleTest() throws Exception
+ {
+ writeLinesToFile("input",
+ "A1\tB1\t1",
+ "A1\tB1\t4",
+ "A1\tB3\t4",
+ "A1\tB4\t4",
+ "A2\tB1\t4",
+ "A2\tB2\t4",
+ "A3\tB1\t3",
+ "A3\tB1\t1",
+ "A3\tB3\t77",
+ "A4\tB1\t3",
+ "A4\tB2\t3",
+ "A4\tB3\t59",
+ "A4\tB4\t29",
+ "A5\tB1\t4",
+ "A6\tB2\t3",
+ "A6\tB2\t55",
+ "A6\tB3\t1",
+ "A7\tB1\t39",
+ "A7\tB2\t27",
+ "A7\tB3\t85",
+ "A8\tB1\t4",
+ "A8\tB2\t45",
+ "A9\tB3\t92",
+ "A9\tB3\t0",
+ "A9\tB6\t42",
+ "A9\tB5\t1",
+ "A10\tB1\t7",
+ "A10\tB2\t23",
+ "A10\tB2\t1",
+ "A10\tB2\t31",
+ "A10\tB6\t41",
+ "A10\tB7\t52");
+
+ int n = 32;
+ double p = 0.3;
+ int s = (int) Math.ceil(p * n);
+ PigTest test =
+ createPigTestFromString(simpleRandomSampleTest, "SAMPLING_PROBABILITY=" + p);
+
+ test.runScript();
+
+ assertOutput(test, "sampled", "(" + s + ")");
+ }
+
+ /**
+ *
+ *
+ * DEFINE SRS datafu.pig.sampling.SimpleRandomSample('$SAMPLING_PROBABILITY');
+ *
+ * data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
+ *
+ * sampled = FOREACH (GROUP data BY A_id) GENERATE group, SRS(data) as sample_data;
+ *
+ * sampled = FOREACH sampled GENERATE group, COUNT(sample_data) AS sample_count;
+ *
+ * sampled = ORDER sampled BY group;
+ *
+ * STORE sampled INTO 'output';
+ */
+ @Multiline
+ private String stratifiedSampleTest;
+
+ @Test
+ public void stratifiedSampleTest() throws Exception
+ {
+ writeLinesToFile("input",
+ "A1\tB1\t1",
+ "A1\tB1\t4",
+ "A1\tB3\t4",
+ "A1\tB4\t4",
+ "A2\tB1\t4",
+ "A2\tB2\t4",
+ "A3\tB1\t3",
+ "A3\tB1\t1",
+ "A3\tB3\t77",
+ "A4\tB1\t3",
+ "A4\tB2\t3",
+ "A4\tB3\t59",
+ "A4\tB4\t29",
+ "A5\tB1\t4",
+ "A6\tB2\t3",
+ "A6\tB2\t55",
+ "A6\tB3\t1",
+ "A7\tB1\t39",
+ "A7\tB2\t27",
+ "A7\tB3\t85",
+ "A8\tB1\t4",
+ "A8\tB2\t45",
+ "A9\tB3\t92",
+ "A9\tB3\t0",
+ "A9\tB6\t42",
+ "A9\tB5\t1",
+ "A10\tB1\t7",
+ "A10\tB2\t23",
+ "A10\tB2\t1",
+ "A10\tB2\t31",
+ "A10\tB6\t41",
+ "A10\tB7\t52");
+
+ double p = 0.5;
+
+ PigTest test =
+ createPigTestFromString(stratifiedSampleTest, "SAMPLING_PROBABILITY=" + p);
+ test.runScript();
+ assertOutput(test,
+ "sampled",
+ "(A1,2)",
+ "(A10,3)",
+ "(A2,1)",
+ "(A3,2)",
+ "(A4,2)",
+ "(A5,1)",
+ "(A6,2)",
+ "(A7,2)",
+ "(A8,1)",
+ "(A9,2)");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/sampling/SimpleRandomSampleWithReplacementTest.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/sampling/SimpleRandomSampleWithReplacementTest.java b/datafu-pig/src/test/java/datafu/test/pig/sampling/SimpleRandomSampleWithReplacementTest.java
new file mode 100644
index 0000000..4132203
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/sampling/SimpleRandomSampleWithReplacementTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.sampling;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.pig.sampling.SimpleRandomSampleWithReplacementElect;
+import datafu.pig.sampling.SimpleRandomSampleWithReplacementVote;
+import datafu.test.pig.PigTests;
+
+/**
+ * Tests for {@link SimpleRandomSampleWithReplacementVote} and
+ * {@link SimpleRandomSampleWithReplacementElect}.
+ *
+ * @author ximeng
+ *
+ */
+public class SimpleRandomSampleWithReplacementTest extends PigTests
+{
+ // @formatter:off
+ /**
+ *
+ *
+ * DEFINE SRSWR_VOTE datafu.pig.sampling.SimpleRandomSampleWithReplacementVote();
+ *
+ * DEFINE SRSWR_ELECT datafu.pig.sampling.SimpleRandomSampleWithReplacementElect();
+ *
+ * item = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
+ *
+ * summary = FOREACH (GROUP item ALL) GENERATE COUNT(item) AS count;
+ *
+ * candidates = FOREACH (GROUP item ALL) GENERATE FLATTEN(SRSWR_VOTE(item, $SAMPLE_SIZE, summary.count));
+ *
+ * sampled = FOREACH (GROUP candidates BY position) GENERATE FLATTEN(SRSWR_ELECT(candidates));
+ *
+ * sampled = FOREACH (GROUP sampled ALL) GENERATE COUNT(sampled) AS size;
+ *
+ * STORE sampled INTO 'output';
+ */
+ @Multiline
+ private String simpleRandomSampleWithReplacementTest;
+ // @formatter:on
+
+ @Test
+ public void testSimpleRandomSampleWithReplacement() throws Exception
+ {
+ writeLinesToFile("input",
+ "A1\tB1\t1",
+ "A1\tB1\t4",
+ "A1\tB3\t4",
+ "A1\tB4\t4",
+ "A2\tB1\t4",
+ "A2\tB2\t4",
+ "A3\tB1\t3",
+ "A3\tB1\t1",
+ "A3\tB3\t77",
+ "A4\tB1\t3",
+ "A4\tB2\t3",
+ "A4\tB3\t59",
+ "A4\tB4\t29",
+ "A5\tB1\t4",
+ "A6\tB2\t3",
+ "A6\tB2\t55",
+ "A6\tB3\t1",
+ "A7\tB1\t39",
+ "A7\tB2\t27",
+ "A7\tB3\t85",
+ "A8\tB1\t4",
+ "A8\tB2\t45",
+ "A9\tB3\t92",
+ "A9\tB3\t0",
+ "A9\tB6\t42",
+ "A9\tB5\t1",
+ "A10\tB1\t7",
+ "A10\tB2\t23",
+ "A10\tB2\t1",
+ "A10\tB2\t31",
+ "A10\tB6\t41",
+ "A10\tB7\t52");
+
+ int s = 32;
+ PigTest test =
+ createPigTestFromString(simpleRandomSampleWithReplacementTest, "SAMPLE_SIZE=" + s
+ + "L");
+
+ test.runScript();
+
+ assertOutput(test, "sampled", "(" + s + ")");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/sampling/WeightedReservoirSamplingTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/sampling/WeightedReservoirSamplingTests.java b/datafu-pig/src/test/java/datafu/test/pig/sampling/WeightedReservoirSamplingTests.java
new file mode 100644
index 0000000..24e7ec7
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/sampling/WeightedReservoirSamplingTests.java
@@ -0,0 +1,373 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.sampling;
+
+import java.io.IOException;
+import java.util.*;
+
+import junit.framework.Assert;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.pig.sampling.WeightedReservoirSample;
+import datafu.test.pig.PigTests;
+
+/**
+ * Tests for {@link WeightedReservoirSample}.
+ *
+ * @author wjian
+ *
+ */
+public class WeightedReservoirSamplingTests extends PigTests
+{
+ /**
+
+
+ DEFINE ReservoirSample datafu.pig.sampling.WeightedReservoirSample('$RESERVOIR_SIZE','2');
+ DEFINE Assert datafu.pig.util.Assert();
+
+ data = LOAD 'input' AS (A_id:int, B_id:chararray, C:int, W:double);
+ sampled = FOREACH (GROUP data BY A_id) GENERATE group as A_id, ReservoirSample(data.(B_id,C,W)) as sample_data;
+ sampled = FILTER sampled BY Assert((SIZE(sample_data) <= $RESERVOIR_SIZE ? 1 : 0), 'must be <= $RESERVOIR_SIZE');
+ sampled = FOREACH sampled GENERATE A_id, FLATTEN(sample_data);
+ STORE sampled INTO 'output';
+
+ */
+ @Multiline
+ private String weightedReservoirSampleGroupTest;
+
+ /**
+ * Verifies that WeightedReservoirSample works when data grouped by a key.
+ * In particular it ensures that the reservoir is not reused across keys.
+ *
+ * <p>
+ * This confirms the fix for DATAFU-11.
+ * </p>
+ *
+ * @throws Exception
+ */
+ @Test
+ public void weightedReservoirSampleGroupTest() throws Exception
+ {
+ // first value is the key. second to last value matches the key so we can
+ // verify the register is reset for each key. values should not
+ // bleed across to other keys.
+ writeLinesToFile("input",
+ "1\tB1\t1\t1.0",
+ "1\tB1\t1\t1.0",
+ "1\tB3\t1\t1.0",
+ "1\tB4\t1\t1.0",
+ "2\tB1\t2\t1.0",
+ "2\tB2\t2\t1.0",
+ "3\tB1\t3\t1.0",
+ "3\tB1\t3\t1.0",
+ "3\tB3\t3\t1.0",
+ "4\tB1\t4\t1.0",
+ "4\tB2\t4\t1.0",
+ "4\tB3\t4\t1.0",
+ "4\tB4\t4\t1.0",
+ "5\tB1\t5\t1.0",
+ "6\tB2\t6\t1.0",
+ "6\tB2\t6\t1.0",
+ "6\tB3\t6\t1.0",
+ "7\tB1\t7\t1.0",
+ "7\tB2\t7\t1.0",
+ "7\tB3\t7\t1.0",
+ "8\tB1\t8\t1.0",
+ "8\tB2\t8\t1.0",
+ "9\tB3\t9\t1.0",
+ "9\tB3\t9\t1.0",
+ "9\tB6\t9\t1.0",
+ "9\tB5\t9\t1.0",
+ "10\tB1\t10\t1.0",
+ "10\tB2\t10\t1.0",
+ "10\tB2\t10\t1.0",
+ "10\tB2\t10\t1.0",
+ "10\tB6\t10\t1.0",
+ "10\tB7\t10\t1.0");
+
+ for(int i=1; i<=3; i++) {
+ int reservoirSize = i ;
+ PigTest test = createPigTestFromString(weightedReservoirSampleGroupTest, "RESERVOIR_SIZE="+reservoirSize);
+ test.runScript();
+
+ List<Tuple> tuples = getLinesForAlias(test, "sampled");
+
+ for (Tuple tuple : tuples)
+ {
+ Assert.assertEquals(((Number)tuple.get(0)).intValue(), ((Number)tuple.get(2)).intValue());
+ }
+ }
+ }
+
+ /**
+
+
+ define WeightedSample datafu.pig.sampling.WeightedReservoirSample('1','1');
+
+ data = LOAD 'input' AS (v1:chararray,v2:INT);
+ data_g = group data all;
+ sampled = FOREACH data_g GENERATE WeightedSample(data);
+ --describe sampled;
+
+ STORE sampled INTO 'output';
+
+ */
+ @Multiline
+ private String weightedSampleTest;
+
+ @Test
+ public void weightedSampleTest() throws Exception
+ {
+ Map<String, Integer> count = new HashMap<String, Integer>();
+
+ count.put("a", 0);
+ count.put("b", 0);
+ count.put("c", 0);
+ count.put("d", 0);
+
+ PigTest test = createPigTestFromString(weightedSampleTest);
+
+ writeLinesToFile("input",
+ "a\t100","b\t1","c\t5","d\t2");
+
+ for(int i = 0; i < 10; i++) {
+ test.runScript();
+
+ List<Tuple> output = this.getLinesForAlias(test, "sampled");
+
+ Tuple t = output.get(0);
+
+ DataBag sampleBag = (DataBag)t.get(0);
+
+ for(Iterator<Tuple> sampleIter = sampleBag.iterator(); sampleIter.hasNext();) {
+ Tuple st = sampleIter.next();
+ String key = (String)st.get(0);
+ count.put(key, count.get(key) + 1);
+ }
+ }
+
+ String maxKey = "";
+ int maxCount = 0;
+ for(String key : count.keySet()) {
+ if(maxCount < count.get(key)) {
+ maxKey = key;
+ maxCount = count.get(key);
+ }
+ }
+
+ Assert.assertEquals(maxKey, "a");
+ }
+
+ @Test
+ public void weightedReservoirSampleAccumulateTest() throws IOException
+ {
+ WeightedReservoirSample sampler = new WeightedReservoirSample("10", "1");
+
+ for (int i=0; i<100; i++)
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(2);
+ t.set(0, i);
+ t.set(1, i + 1);
+ DataBag bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(t);
+ Tuple input = TupleFactory.getInstance().newTuple(bag);
+ sampler.accumulate(input);
+ }
+
+ DataBag result = sampler.getValue();
+ verifyNoRepeatAllFound(result, 10, 0, 100);
+ }
+
+ @Test
+ public void weightedReservoirSampleAlgebraicTest() throws IOException
+ {
+ WeightedReservoirSample.Initial initialSampler = new WeightedReservoirSample.Initial("10", "1");
+ WeightedReservoirSample.Intermediate intermediateSampler = new WeightedReservoirSample.Intermediate("10", "1");
+ WeightedReservoirSample.Final finalSampler = new WeightedReservoirSample.Final("10", "1");
+
+ DataBag bag = BagFactory.getInstance().newDefaultBag();
+ for (int i=0; i<100; i++)
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(2);
+ t.set(0, i);
+ t.set(1, i + 1);
+ bag.add(t);
+ }
+
+ Tuple input = TupleFactory.getInstance().newTuple(bag);
+
+ Tuple intermediateTuple = initialSampler.exec(input);
+ DataBag intermediateBag = BagFactory.getInstance().newDefaultBag(Arrays.asList(intermediateTuple));
+ intermediateTuple = intermediateSampler.exec(TupleFactory.getInstance().newTuple(intermediateBag));
+ intermediateBag = BagFactory.getInstance().newDefaultBag(Arrays.asList(intermediateTuple));
+ DataBag result = finalSampler.exec(TupleFactory.getInstance().newTuple(intermediateBag));
+ verifyNoRepeatAllFound(result, 10, 0, 100);
+ }
+
+ private void verifyNoRepeatAllFound(DataBag result,
+ int expectedResultSize,
+ int left,
+ int right) throws ExecException
+ {
+ Assert.assertEquals(expectedResultSize, result.size());
+
+ // all must be found, no repeats
+ Set<Integer> found = new HashSet<Integer>();
+ for (Tuple t : result)
+ {
+ Integer i = (Integer)t.get(0);
+ Assert.assertTrue(i>=left && i<right);
+ Assert.assertFalse(String.format("Found duplicate of %d",i), found.contains(i));
+ found.add(i);
+ }
+ }
+
+ @Test
+ public void weightedReservoirSampleLimitExecTest() throws IOException
+ {
+ WeightedReservoirSample sampler = new WeightedReservoirSample("100", "1");
+
+ DataBag bag = BagFactory.getInstance().newDefaultBag();
+ for (int i=0; i<10; i++)
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(2);
+ t.set(0, i);
+ t.set(1, 1); // score is equal for all
+ bag.add(t);
+ }
+
+ Tuple input = TupleFactory.getInstance().newTuple(1);
+ input.set(0, bag);
+
+ DataBag result = sampler.exec(input);
+
+ verifyNoRepeatAllFound(result, 10, 0, 10);
+
+ Set<Integer> found = new HashSet<Integer>();
+ for (Tuple t : result)
+ {
+ Integer i = (Integer)t.get(0);
+ found.add(i);
+ }
+
+ for(int i = 0; i < 10; i++)
+ {
+ Assert.assertTrue(found.contains(i));
+ }
+ }
+
+ @Test
+ public void invalidConstructorArgTest() throws Exception
+ {
+ try {
+ WeightedReservoirSample sampler = new WeightedReservoirSample("1", "-1");
+ Assert.fail( "Testcase should fail");
+ } catch (Exception ex) {
+ Assert.assertTrue(ex.getMessage().indexOf("Invalid negative index of weight field argument for WeightedReserviorSample constructor: -1") >= 0);
+ }
+ }
+
+ @Test
+ public void invalidWeightTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(weightedSampleTest);
+
+ writeLinesToFile("input",
+ "a\t100","b\t1","c\t0","d\t2");
+ try {
+ test.runScript();
+ List<Tuple> output = this.getLinesForAlias(test, "sampled");
+ Assert.fail( "Testcase should fail");
+ } catch (Exception ex) {
+ }
+ }
+
+ /**
+
+
+ define WeightedSample datafu.pig.sampling.WeightedReservoirSample('1','1');
+
+ data = LOAD 'input' AS (v1:chararray);
+ data_g = group data all;
+ sampled = FOREACH data_g GENERATE WeightedSample(data);
+ describe sampled;
+
+ STORE sampled INTO 'output';
+
+ */
+ @Multiline
+ private String invalidInputTupleSizeTest;
+
+ @Test
+ public void invalidInputTupleSizeTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(invalidInputTupleSizeTest);
+
+ writeLinesToFile("input",
+ "a","b","c","d");
+ try {
+ test.runScript();
+ List<Tuple> output = this.getLinesForAlias(test, "sampled");
+ Assert.fail( "Testcase should fail");
+ } catch (Exception ex) {
+ Assert.assertTrue(ex.getMessage().indexOf("The field schema of the input tuple is null or the tuple size is no more than the weight field index: 1") >= 0);
+ }
+ }
+
+ /**
+
+
+ define WeightedSample datafu.pig.sampling.WeightedReservoirSample('1','0');
+
+ data = LOAD 'input' AS (v1:chararray, v2:INT);
+ data_g = group data all;
+ sampled = FOREACH data_g GENERATE WeightedSample(data);
+ describe sampled;
+
+ STORE sampled INTO 'output';
+
+ */
+ @Multiline
+ private String invalidWeightFieldSchemaTest;
+
+ @Test
+ public void invalidWeightFieldSchemaTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(invalidWeightFieldSchemaTest);
+
+ writeLinesToFile("input",
+ "a\t100","b\t1","c\t5","d\t2");
+ try {
+ test.runScript();
+ List<Tuple> output = this.getLinesForAlias(test, "sampled");
+ Assert.fail( "Testcase should fail");
+ } catch (Exception ex) {
+ Assert.assertTrue(ex.getMessage().indexOf("Expect the type of the weight field of the input tuple to be of ([int, long, float, double]), but instead found (chararray), weight field: 0") >= 0);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/sessions/SessionTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/sessions/SessionTests.java b/datafu-pig/src/test/java/datafu/test/pig/sessions/SessionTests.java
new file mode 100644
index 0000000..76ec1d3
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/sessions/SessionTests.java
@@ -0,0 +1,430 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.sessions;
+
+import static org.testng.Assert.*;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.commons.lang.StringUtils;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.pigunit.PigTest;
+import org.joda.time.DateTime;
+import org.testng.annotations.Test;
+
+import datafu.pig.sessions.SessionCount;
+import datafu.pig.sessions.Sessionize;
+import datafu.test.pig.PigTests;
+
+public class SessionTests extends PigTests
+{
+ /**
+
+
+ define Sessionize datafu.pig.sessions.Sessionize('$TIME_WINDOW');
+
+ views = LOAD 'input' AS (time:$TIME_TYPE, user_id:int, value:int);
+
+ views_grouped = GROUP views BY user_id;
+ view_counts = FOREACH views_grouped {
+ views = ORDER views BY time;
+ GENERATE flatten(Sessionize(views)) as (time,user_id,value,session_id);
+ }
+
+ max_value = GROUP view_counts BY (user_id, session_id);
+
+ max_value = FOREACH max_value GENERATE group.user_id, MAX(view_counts.value) AS val;
+
+ STORE max_value INTO 'output';
+ */
+ @Multiline private String sessionizeTest;
+
+ private String[] inputData = new String[] {
+ "2010-01-01T01:00:00Z\t1\t10",
+ "2010-01-01T01:15:00Z\t1\t20",
+ "2010-01-01T01:31:00Z\t1\t10",
+ "2010-01-01T01:35:00Z\t1\t20",
+ "2010-01-01T02:30:00Z\t1\t30",
+
+ "2010-01-01T01:00:00Z\t2\t10",
+ "2010-01-01T01:31:00Z\t2\t20",
+ "2010-01-01T02:10:00Z\t2\t30",
+ "2010-01-01T02:40:30Z\t2\t40",
+ "2010-01-01T03:30:00Z\t2\t50",
+
+ "2010-01-01T01:00:00Z\t3\t10",
+ "2010-01-01T01:01:00Z\t3\t20",
+ "2010-01-01T01:02:00Z\t3\t5",
+ "2010-01-01T01:10:00Z\t3\t25",
+ "2010-01-01T01:15:00Z\t3\t50",
+ "2010-01-01T01:25:00Z\t3\t30",
+ "2010-01-01T01:30:00Z\t3\t15"
+ };
+
+ @Test
+ public void sessionizeTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(sessionizeTest,
+ "TIME_WINDOW=30m",
+ "TIME_TYPE=chararray");
+
+ this.writeLinesToFile("input",
+ inputData);
+
+ test.runScript();
+
+ HashMap<Integer,HashMap<Integer,Boolean>> userValues = new HashMap<Integer,HashMap<Integer,Boolean>>();
+
+ for (Tuple t : this.getLinesForAlias(test, "max_value"))
+ {
+ Integer userId = (Integer)t.get(0);
+ Integer max = (Integer)t.get(1);
+ if (!userValues.containsKey(userId))
+ {
+ userValues.put(userId, new HashMap<Integer,Boolean>());
+ }
+ userValues.get(userId).put(max, true);
+ }
+
+ assertEquals(userValues.get(1).size(), 2);
+ assertEquals(userValues.get(2).size(), 5);
+ assertEquals(userValues.get(3).size(), 1);
+
+ assertTrue(userValues.get(1).containsKey(20));
+ assertTrue(userValues.get(1).containsKey(30));
+
+ assertTrue(userValues.get(2).containsKey(10));
+ assertTrue(userValues.get(2).containsKey(20));
+ assertTrue(userValues.get(2).containsKey(30));
+ assertTrue(userValues.get(2).containsKey(40));
+ assertTrue(userValues.get(2).containsKey(50));
+
+ assertTrue(userValues.get(3).containsKey(50));
+ }
+
+ private SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
+
+ @Test
+ public void sessionizeLongTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(sessionizeTest,
+ "TIME_WINDOW=30m",
+ "TIME_TYPE=long");
+
+ List<String> lines = new ArrayList<String>();
+
+ for (String line : inputData)
+ {
+ String[] parts = line.split("\t");
+ Assert.assertEquals(3, parts.length);
+ parts[0] = Long.toString(dateFormat.parse(parts[0]).getTime());
+ lines.add(StringUtils.join(parts,"\t"));
+ }
+
+ this.writeLinesToFile("input",
+ lines.toArray(new String[]{}));
+
+ test.runScript();
+
+ HashMap<Integer,HashMap<Integer,Boolean>> userValues = new HashMap<Integer,HashMap<Integer,Boolean>>();
+
+ for (Tuple t : this.getLinesForAlias(test, "max_value"))
+ {
+ Integer userId = (Integer)t.get(0);
+ Integer max = (Integer)t.get(1);
+ if (!userValues.containsKey(userId))
+ {
+ userValues.put(userId, new HashMap<Integer,Boolean>());
+ }
+ userValues.get(userId).put(max, true);
+ }
+
+ assertEquals(userValues.get(1).size(), 2);
+ assertEquals(userValues.get(2).size(), 5);
+
+ assertTrue(userValues.get(1).containsKey(20));
+ assertTrue(userValues.get(1).containsKey(30));
+
+ assertTrue(userValues.get(2).containsKey(10));
+ assertTrue(userValues.get(2).containsKey(20));
+ assertTrue(userValues.get(2).containsKey(30));
+ assertTrue(userValues.get(2).containsKey(40));
+ assertTrue(userValues.get(2).containsKey(50));
+ }
+
+ @Test
+ public void sessionizeExecTest() throws Exception
+ {
+ Sessionize sessionize = new Sessionize("30m");
+ Tuple input = TupleFactory.getInstance().newTuple(1);
+ DataBag inputBag = BagFactory.getInstance().newDefaultBag();
+ input.set(0,inputBag);
+
+ Tuple item;
+ List<Tuple> result;
+ DateTime dt;
+
+ // test same session id
+ inputBag.clear();
+ dt = new DateTime();
+ item = TupleFactory.getInstance().newTuple(1);
+ item.set(0, dt.getMillis());
+ inputBag.add(item);
+ item = TupleFactory.getInstance().newTuple(1);
+ item.set(0, dt.plusMinutes(28).getMillis());
+ inputBag.add(item);
+ result = toList(sessionize.exec(input));
+
+ Assert.assertEquals(2, result.size());
+ Assert.assertEquals(2,result.get(0).size());
+ Assert.assertEquals(2,result.get(1).size());
+ // session ids match
+ Assert.assertTrue(result.get(0).get(1).equals(result.get(1).get(1)));
+
+ // test different session id
+ inputBag.clear();
+ dt = new DateTime();
+ item = TupleFactory.getInstance().newTuple(1);
+ item.set(0, dt.getMillis());
+ inputBag.add(item);
+ item = TupleFactory.getInstance().newTuple(1);
+ item.set(0, dt.plusMinutes(31).getMillis());
+ inputBag.add(item);
+ result = toList(sessionize.exec(input));
+
+ Assert.assertEquals(2, result.size());
+ Assert.assertEquals(2,result.get(0).size());
+ Assert.assertEquals(2,result.get(1).size());
+ // session ids don't match
+ Assert.assertFalse(result.get(0).get(1).equals(result.get(1).get(1)));
+ }
+
+ @Test
+ public void sessionizeAccumulateTest() throws Exception
+ {
+ Sessionize sessionize = new Sessionize("30m");
+ Tuple input = TupleFactory.getInstance().newTuple(1);
+ DataBag inputBag = BagFactory.getInstance().newDefaultBag();
+ input.set(0,inputBag);
+
+ Tuple item;
+ List<Tuple> result;
+ DateTime dt;
+
+ // test same session id
+ dt = new DateTime();
+ item = TupleFactory.getInstance().newTuple(1);
+ item.set(0, dt.getMillis());
+ inputBag.add(item);
+ sessionize.accumulate(input);
+ inputBag.clear();
+ item = TupleFactory.getInstance().newTuple(1);
+ item.set(0, dt.plusMinutes(28).getMillis());
+ inputBag.add(item);
+ sessionize.accumulate(input);
+ inputBag.clear();
+ result = toList(sessionize.getValue());
+
+ Assert.assertEquals(2, result.size());
+ Assert.assertEquals(2,result.get(0).size());
+ Assert.assertEquals(2,result.get(1).size());
+ // session ids match
+ Assert.assertTrue(result.get(0).get(1).equals(result.get(1).get(1)));
+
+ // test different session id
+ sessionize.cleanup();
+ dt = new DateTime();
+ item = TupleFactory.getInstance().newTuple(1);
+ item.set(0, dt.getMillis());
+ inputBag.add(item);
+ sessionize.accumulate(input);
+ inputBag.clear();
+ item = TupleFactory.getInstance().newTuple(1);
+ item.set(0, dt.plusMinutes(31).getMillis());
+ inputBag.add(item);
+ sessionize.accumulate(input);
+ inputBag.clear();
+ result = toList(sessionize.getValue());
+
+ Assert.assertEquals(2, result.size());
+ Assert.assertEquals(2,result.get(0).size());
+ Assert.assertEquals(2,result.get(1).size());
+ // session ids don't match
+ Assert.assertFalse(result.get(0).get(1).equals(result.get(1).get(1)));
+
+ sessionize.cleanup();
+ Assert.assertEquals(0,sessionize.getValue().size());
+ }
+
+ private List<Tuple> toList(DataBag bag)
+ {
+ List<Tuple> result = new ArrayList<Tuple>();
+ for (Tuple t : bag)
+ {
+ result.add(t);
+ }
+ return result;
+ }
+
+ /**
+
+
+ define SessionCount datafu.pig.sessions.SessionCount('$TIME_WINDOW');
+
+ views = LOAD 'input' AS (user_id:int, page_id:int, time:chararray);
+
+ views_grouped = GROUP views BY (user_id, page_id);
+ view_counts = foreach views_grouped {
+ views = order views by time;
+ generate group.user_id as user_id, group.page_id as page_id, SessionCount(views.(time)) as count;
+ }
+
+ STORE view_counts INTO 'output';
+ */
+ @Multiline
+ private String sessionCountPageViewsTest;
+
+ @Test
+ public void sessionCountPageViewsTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(sessionCountPageViewsTest,
+ "TIME_WINDOW=30m");
+
+ String[] input = {
+ "1\t100\t2010-01-01T01:00:00Z",
+ "1\t100\t2010-01-01T01:15:00Z",
+ "1\t100\t2010-01-01T01:31:00Z",
+ "1\t100\t2010-01-01T01:35:00Z",
+ "1\t100\t2010-01-01T02:30:00Z",
+
+ "1\t101\t2010-01-01T01:00:00Z",
+ "1\t101\t2010-01-01T01:31:00Z",
+ "1\t101\t2010-01-01T02:10:00Z",
+ "1\t101\t2010-01-01T02:40:30Z",
+ "1\t101\t2010-01-01T03:30:00Z",
+
+ "1\t102\t2010-01-01T01:00:00Z",
+ "1\t102\t2010-01-01T01:01:00Z",
+ "1\t102\t2010-01-01T01:02:00Z",
+ "1\t102\t2010-01-01T01:10:00Z",
+ "1\t102\t2010-01-01T01:15:00Z",
+ "1\t102\t2010-01-01T01:25:00Z",
+ "1\t102\t2010-01-01T01:30:00Z"
+ };
+
+ String[] output = {
+ "(1,100,2)",
+ "(1,101,5)",
+ "(1,102,1)"
+ };
+
+ test.assertOutput("views",input,"view_counts",output);
+ }
+
+ @Test
+ public void sessionCountExecTest() throws Exception
+ {
+ SessionCount sessionize = new SessionCount("30m");
+ Tuple input = TupleFactory.getInstance().newTuple(1);
+ DataBag inputBag = BagFactory.getInstance().newDefaultBag();
+ input.set(0,inputBag);
+
+ Tuple item;
+ DateTime dt;
+
+ // test same session id
+ inputBag.clear();
+ dt = new DateTime();
+ item = TupleFactory.getInstance().newTuple(1);
+ item.set(0, dt.getMillis());
+ inputBag.add(item);
+ item = TupleFactory.getInstance().newTuple(1);
+ item.set(0, dt.plusMinutes(28).getMillis());
+ inputBag.add(item);
+ Assert.assertEquals(1L,sessionize.exec(input).longValue());
+
+ // test different session id
+ inputBag.clear();
+ dt = new DateTime();
+ item = TupleFactory.getInstance().newTuple(1);
+ item.set(0, dt.getMillis());
+ inputBag.add(item);
+ item = TupleFactory.getInstance().newTuple(1);
+ item.set(0, dt.plusMinutes(31).getMillis());
+ inputBag.add(item);
+ Assert.assertEquals(2L,sessionize.exec(input).longValue());
+ }
+
+ @Test
+ public void sessionCountAccumulateTest() throws Exception
+ {
+ SessionCount sessionize = new SessionCount("30m");
+ Tuple input = TupleFactory.getInstance().newTuple(1);
+ DataBag inputBag = BagFactory.getInstance().newDefaultBag();
+ input.set(0,inputBag);
+
+ Tuple item;
+ DateTime dt;
+
+ // test same session id
+ dt = new DateTime();
+ item = TupleFactory.getInstance().newTuple(1);
+ item.set(0, dt.getMillis());
+ inputBag.add(item);
+ sessionize.accumulate(input);
+ inputBag.clear();
+ item = TupleFactory.getInstance().newTuple(1);
+ item.set(0, dt.plusMinutes(28).getMillis());
+ inputBag.add(item);
+ sessionize.accumulate(input);
+ inputBag.clear();
+ Assert.assertEquals(1L,sessionize.getValue().longValue());
+
+ // test different session id
+ sessionize.cleanup();
+ dt = new DateTime();
+ item = TupleFactory.getInstance().newTuple(1);
+ item.set(0, dt.getMillis());
+ inputBag.add(item);
+ sessionize.accumulate(input);
+ inputBag.clear();
+ item = TupleFactory.getInstance().newTuple(1);
+ item.set(0, dt.plusMinutes(31).getMillis());
+ inputBag.add(item);
+ sessionize.accumulate(input);
+ inputBag.clear();
+ Assert.assertEquals(2L,sessionize.exec(input).longValue());
+
+ sessionize.cleanup();
+ Assert.assertEquals(0,sessionize.getValue().longValue());
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/sets/SetTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/sets/SetTests.java b/datafu-pig/src/test/java/datafu/test/pig/sets/SetTests.java
new file mode 100644
index 0000000..9b24797
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/sets/SetTests.java
@@ -0,0 +1,330 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.sets;
+
+import java.util.Arrays;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import datafu.pig.sets.SetIntersect;
+import datafu.test.pig.PigTests;
+
+public class SetTests extends PigTests
+{
+ /**
+
+
+ define SetIntersect datafu.pig.sets.SetIntersect();
+
+ data = LOAD 'input' AS (B1:bag{T:tuple(val1:int,val2:int)},B2:bag{T:tuple(val1:int,val2:int)});
+
+ data2 = FOREACH data GENERATE SetIntersect(B1,B2);
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline
+ private String setIntersectTest;
+
+ @Test
+ public void setIntersectTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(setIntersectTest);
+
+ writeLinesToFile("input",
+ "{(1,10),(2,20),(3,30),(4,40),(5,50),(6,60)}\t{(0,0),(2,20),(4,40),(8,80)}",
+ "{(1,10),(1,10),(2,20),(3,30),(3,30),(4,40),(4,40)}\t{(1,10),(3,30)}");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "({(2,20),(4,40)})",
+ "({(1,10),(3,30)})");
+ }
+
+ /**
+
+
+ define SetIntersect datafu.pig.sets.SetIntersect();
+
+ docs = LOAD 'docs' USING PigStorage(',') AS (id:int, line:chararray);
+ B = FOREACH docs GENERATE id, line;
+ C = FOREACH B GENERATE id, TOKENIZE(line) as gu;
+
+ filtered = FOREACH C {
+ uniq = DISTINCT gu;
+ GENERATE id, uniq;
+ }
+
+ query = LOAD 'query' AS (line_query:chararray);
+ bag_query = FOREACH query GENERATE TOKENIZE(line_query) AS query;
+ -- sort the bag of tokens, since SetIntersect requires it
+ bag_query = FOREACH bag_query {
+ query_sorted = ORDER query BY token;
+ GENERATE query_sorted;
+ }
+
+ result = FOREACH filtered {
+ -- sort the tokens, since SetIntersect requires it
+ tokens_sorted = ORDER uniq BY token;
+ GENERATE id,
+ SIZE(SetIntersect(tokens_sorted,bag_query.query_sorted)) as cnt;
+ }
+
+ DUMP result;
+
+ */
+ @Multiline
+ private String setIntersectTestExample;
+
+ @Test
+ public void setIntersectTestExample() throws Exception
+ {
+ PigTest test = createPigTestFromString(setIntersectTestExample);
+
+ // document with words to check for
+ writeLinesToFile("docs",
+ "1,word1 word4 word2 word1",
+ "2,word2 word6 word1 word5 word3 word7",
+ "3,word1 word3 word4 word5");
+
+ // query to apply to document
+ writeLinesToFile("query",
+ "word2 word7 word5");
+
+ test.runScript();
+
+ assertOutput(test, "result",
+ "(1,1)",
+ "(2,3)",
+ "(3,1)");
+ }
+
+ @Test
+ public void setIntersectEmptyBagsTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(setIntersectTest);
+
+ writeLinesToFile("input",
+ "{(1,10),(2,20),(3,30),(4,40),(5,50),(6,60)}\t{(0,0),(2,20),(4,40),(8,80)}",
+ "{(1,10),(1,10),(2,20),(3,30),(3,30),(4,40),(4,40)}\t{(100,10),(300,30)}",
+ "{(1,10),(2,20)}\t{(1,10),(2,20)}",
+ "{(1,10),(2,20)}\t{}",
+ "{}\t{(1,10),(2,20)}",
+ "{}\t{}");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "({(2,20),(4,40)})",
+ "({})",
+ "({(1,10),(2,20)})",
+ "({})",
+ "({})",
+ "({})");
+ }
+
+ @Test
+ public void testIntersectWithNullTuples() throws Exception {
+ DataBag one = BagFactory.getInstance().newDefaultBag();
+ DataBag two = BagFactory.getInstance().newDefaultBag();
+
+ Tuple input = TupleFactory.getInstance().newTuple(Arrays.asList(one, two));
+ DataBag output = new SetIntersect().exec(input);
+ Assert.assertEquals(0, output.size());
+ }
+
+ @Test(expectedExceptions=org.apache.pig.impl.logicalLayer.FrontendException.class)
+ public void setIntersectOutOfOrderTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(setIntersectTest);
+
+ this.writeLinesToFile("input",
+ "{(1,10),(3,30),(2,20),(4,40),(5,50),(6,60)}\t{(0,0),(2,20),(4,40),(8,80)}");
+
+ test.runScript();
+
+ this.getLinesForAlias(test, "data2");
+ }
+
+ /**
+
+
+ define SetUnion datafu.pig.sets.SetUnion();
+
+ data = LOAD 'input' AS (B1:bag{T:tuple(val1:int,val2:int)},B2:bag{T:tuple(val1:int,val2:int)});
+
+ --dump data
+
+ data2 = FOREACH data GENERATE SetUnion(B1,B2) AS C;
+ data2 = FOREACH data2 {
+ C = ORDER C BY val1 ASC, val2 ASC;
+ generate C;
+ }
+
+ --dump data2
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline
+ private String setUnionTest;
+
+ @Test
+ public void setUnionTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(setUnionTest);
+
+ writeLinesToFile("input",
+ "{(1,10),(1,20),(1,30),(1,40),(1,50),(1,60),(1,80)}\t{(1,1),(1,20),(1,25),(1,25),(1,25),(1,40),(1,70),(1,80)}");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "({(1,1),(1,10),(1,20),(1,25),(1,30),(1,40),(1,50),(1,60),(1,70),(1,80)})");
+ }
+
+ @Test
+ public void setUnionEmptyBagsTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(setUnionTest);
+
+ writeLinesToFile("input",
+ "{(1,10),(1,20),(1,30),(1,40),(1,50),(1,60),(1,80)}\t{}",
+ "{}\t{(1,10),(1,20),(1,30),(1,40),(1,50)}",
+ "{}\t{}");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "({(1,10),(1,20),(1,30),(1,40),(1,50),(1,60),(1,80)})",
+ "({(1,10),(1,20),(1,30),(1,40),(1,50)})",
+ "({})");
+ }
+
+ /**
+
+
+ define SetDifference datafu.pig.sets.SetDifference();
+
+ data = LOAD 'input' AS (B1:bag{T:tuple(val:int)},B2:bag{T:tuple(val:int)});
+
+ data2 = FOREACH data GENERATE SetDifference(B1,B2);
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline
+ private String setDifferenceTwoBagsTest;
+
+ @Test
+ public void setDifferenceTwoBagsTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(setDifferenceTwoBagsTest);
+
+ writeLinesToFile("input",
+ "{(1),(2),(3)}\t",
+ "{(1),(2),(3)}\t{}",
+ "\t{(1),(2),(3)}",
+ "{}\t{(1),(2),(3)}",
+ "{(1),(2),(3)}\t{(1)}",
+ "{(1),(2),(3)}\t{(1),(2)}",
+ "{(1),(2),(3)}\t{(1),(2),(3)}",
+ "{(1),(2),(3)}\t{(1),(2),(3),(4)}",
+ "{(1),(2),(3),(4),(5),(6)}\t{(3),(4)}");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "({(1),(2),(3)})",
+ "({(1),(2),(3)})",
+ "({})",
+ "({})",
+ "({(2),(3)})",
+ "({(3)})",
+ "({})",
+ "({})",
+ "({(1),(2),(5),(6)})");
+ }
+
+ /**
+
+
+ define SetDifference datafu.pig.sets.SetDifference();
+
+ data = LOAD 'input' AS (B1:bag{T:tuple(val:int)},B2:bag{T:tuple(val:int)},B3:bag{T:tuple(val:int)});
+
+ data2 = FOREACH data GENERATE SetDifference(B1,B2,B3);
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline
+ private String setDifferenceThreeBagsTest;
+
+ @Test
+ public void setDifferenceThreeBagsTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(setDifferenceThreeBagsTest);
+
+ writeLinesToFile("input",
+ "{(1),(2),(3)}\t\t",
+ "{(1),(2),(3)}\t{}\t",
+ "{(1),(2),(3)}\t\t{}",
+ "{(1),(2),(3)}\t{}\t{}",
+ "{(1),(2),(2),(2),(3),(3)}\t{}\t{}",
+
+ "{(1),(2),(3)}\t{(2)}\t{}",
+ "{(1),(2),(3)}\t{}\t{(2)}",
+
+ "{(1),(2),(3)}\t{(2)}\t{(3)}",
+
+ "{(1),(2),(3)}\t{(2),(2)}\t{(3),(3)}",
+ "{(1),(1),(1),(2),(2),(3),(3),(3)}\t{(2),(2)}\t{(3),(3)}",
+ "{(1),(2),(3)}\t{(0),(2)}\t{(3)}",
+
+ "{(1),(2),(3)}\t{(0),(2)}\t{(1),(3)}",
+ "{(1),(2),(3)}\t{(0),(2)}\t{(1),(3),(4)}");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "({(1),(2),(3)})",
+ "({(1),(2),(3)})",
+ "({(1),(2),(3)})",
+ "({(1),(2),(3)})",
+ "({(1),(2),(3)})",
+
+ "({(1),(3)})",
+ "({(1),(3)})",
+
+ "({(1)})",
+
+ "({(1)})",
+ "({(1)})",
+ "({(1)})",
+
+ "({})",
+ "({})");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/stats/EstimationTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/stats/EstimationTests.java b/datafu-pig/src/test/java/datafu/test/pig/stats/EstimationTests.java
new file mode 100644
index 0000000..174da84
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/stats/EstimationTests.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.stats;
+
+import java.util.List;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.test.pig.PigTests;
+import static org.testng.Assert.*;
+
+public class EstimationTests extends PigTests
+{
+ /**
+
+
+ define HyperLogLogPlusPlus datafu.pig.stats.HyperLogLogPlusPlus();
+
+ data_in = LOAD 'input' as (val:int);
+
+ data_out = FOREACH (GROUP data_in ALL) GENERATE
+ HyperLogLogPlusPlus(data_in) as cardinality;
+
+ data_out = FOREACH data_out GENERATE cardinality;
+
+ STORE data_out into 'output';
+ */
+ @Multiline private String hyperLogLogTest;
+
+ @Test
+ public void hyperLogLogTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(hyperLogLogTest);
+
+ int count = 1000000;
+ String[] input = new String[count];
+ for (int i=0; i<count; i++)
+ {
+ input[i] = Integer.toString(i*10);
+ }
+
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ double error = Math.abs(count-((Long)output.get(0).get(0)))/(double)count;
+ System.out.println("error: " + error*100.0 + "%");
+ assertTrue(error < 0.01);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/stats/MarkovPairTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/stats/MarkovPairTests.java b/datafu-pig/src/test/java/datafu/test/pig/stats/MarkovPairTests.java
new file mode 100644
index 0000000..5d701a7
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/stats/MarkovPairTests.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.stats;
+
+import static org.testng.Assert.*;
+
+import java.util.Iterator;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.test.pig.PigTests;
+
+public class MarkovPairTests extends PigTests
+{
+ /**
+
+
+ define markovPairs datafu.pig.stats.MarkovPairs();
+
+ data = load 'input' as $schema;
+ --describe data;
+
+ data_out1 = foreach data generate data as orig_bag;
+ --describe data_out1;
+
+ data_out = foreach data_out1 generate markovPairs(orig_bag) as markov_bag;
+ --describe data_out;
+
+ store data_out into 'output';
+ */
+ @Multiline private String markovPairDefault;
+
+ @Test
+ public void markovPairDefaultTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(markovPairDefault,
+ "schema=(data: bag {t: tuple(val:int)})");
+
+ writeLinesToFile("input", "{(10),(20),(30),(40),(50),(60)}");
+
+ String[] expectedOutput = {
+ "({((10),(20)),((20),(30)),((30),(40)),((40),(50)),((50),(60))})"
+ };
+
+ test.runScript();
+
+ Iterator<Tuple> actualOutput = test.getAlias("data_out");
+
+ assertTuplesMatch(expectedOutput, actualOutput);
+ }
+
+ @Test
+ public void markovPairMultipleInput() throws Exception
+ {
+ PigTest test = createPigTestFromString(markovPairDefault,
+ "schema=(data: bag {t: tuple(val1:int,val2:int)})");
+
+ writeLinesToFile("input", "{(10,100),(20,200),(30,300),(40,400),(50,500),(60,600)}");
+
+ String[] expectedOutput = {
+ "({((10,100),(20,200)),((20,200),(30,300)),((30,300),(40,400)),((40,400),(50,500)),((50,500),(60,600))})"
+ };
+
+
+ test.runScript();
+
+ Iterator<Tuple> actualOutput = test.getAlias("data_out");
+
+ assertTuplesMatch(expectedOutput, actualOutput);
+ }
+
+ /**
+
+
+ define markovPairs datafu.pig.stats.MarkovPairs('$lookahead');
+
+ data = load 'input' as $schema;
+ --describe data;
+
+ data_out1 = foreach data generate data as orig_bag;
+ --describe data_out1;
+
+ data_out = foreach data_out1 generate markovPairs(orig_bag) as markov_bag;
+ --describe data_out;
+
+ store data_out into 'output';
+ */
+ @Multiline private String markovPairLookahead;
+
+ @Test
+ public void markovPairLookaheadTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(markovPairLookahead,
+ "schema=(data: bag {t: tuple(val:int)})",
+ "lookahead=3");
+
+ writeLinesToFile("input", "{(10),(20),(30),(40),(50)}");
+
+ String[] expectedOutput = {
+ "({((10),(20)),((10),(30)),((10),(40)),((20),(30)),((20),(40)),((20),(50)),((30),(40)),((30),(50)),((40),(50))})"
+ };
+
+ test.runScript();
+
+ Iterator<Tuple> actualOutput = test.getAlias("data_out");
+
+ assertTuplesMatch(expectedOutput, actualOutput);
+ }
+
+ private void assertTuplesMatch(String[] expectedOutput, Iterator<Tuple> actualOutput)
+ {
+ Iterator<Tuple> tuples = actualOutput;
+
+ for (String outputLine : expectedOutput)
+ {
+ assertTrue(tuples.hasNext());
+ Tuple outputTuple = tuples.next();
+ System.out.println(String.format("expected: %s", outputLine));
+ System.out.println(String.format("actual: %s", outputTuple.toString()));
+ assertEquals(outputLine,outputTuple.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/stats/QuantileTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/stats/QuantileTests.java b/datafu-pig/src/test/java/datafu/test/pig/stats/QuantileTests.java
new file mode 100644
index 0000000..b1940db
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/stats/QuantileTests.java
@@ -0,0 +1,696 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.stats;
+
+import static org.testng.Assert.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.pig.stats.QuantileUtil;
+import datafu.pig.stats.StreamingQuantile;
+import datafu.test.pig.PigTests;
+
+public class QuantileTests extends PigTests
+{
+ /**
+
+
+ define Quantile datafu.pig.stats.Quantile($QUANTILES);
+
+ data_in = LOAD 'input' as (val:int);
+
+ --describe data_in;
+
+ data_out = GROUP data_in ALL;
+
+ --describe data_out;
+
+ data_out = FOREACH data_out {
+ sorted = ORDER data_in BY val;
+ GENERATE Quantile(sorted) as quantiles;
+ }
+ data_out = FOREACH data_out GENERATE FLATTEN(quantiles);
+
+ --describe data_out;
+
+ STORE data_out into 'output';
+ */
+ @Multiline private String quantileTest;
+
+ @Test
+ public void quantileTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(quantileTest,
+ "QUANTILES='0.0','0.25','0.5','0.75','1.0'");
+
+ String[] input = {"1","2","3","4","10","5","6","7","8","9"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(1.0,3.0,5.5,8.0,10.0)");
+ }
+
+ @Test
+ public void quantile2Test() throws Exception
+ {
+ PigTest test = createPigTestFromString(quantileTest,
+ "QUANTILES='5'");
+
+ String[] input = {"1","2","3","4","10","5","6","7","8","9"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(1.0,3.0,5.5,8.0,10.0)");
+ }
+
+ @Test
+ public void quantile3Test() throws Exception {
+ PigTest test = createPigTestFromString(quantileTest,
+ "QUANTILES='0.0013','0.0228','0.1587','0.5','0.8413','0.9772','0.9987'");
+
+ List<String> input = new ArrayList<String>();
+ for (int i=100000; i>=0; i--)
+ {
+ input.add(Integer.toString(i));
+ }
+
+ writeLinesToFile("input", input.toArray(new String[0]));
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(130.0,2280.0,15870.0,50000.0,84130.0,97720.0,99870.0)");
+ }
+
+ @Test
+ public void quantile4aTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(quantileTest,
+ "QUANTILES='4'");
+
+ String[] input = {"1","2","3","4","10","5","6","7","8","9"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(1.0,4.0,7.0,10.0)");
+ }
+
+ @Test
+ public void quantile4bTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(quantileTest,
+ "QUANTILES='0.0','0.333','0.666','1.0'");
+
+ String[] input = {"1","2","3","4","10","5","6","7","8","9"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(1.0,4.0,7.0,10.0)");
+ }
+
+ @Test
+ public void quantile5aTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(quantileTest,
+ "QUANTILES='10'");
+
+ String[] input = {"1","2","3","4","10","5","6","7","8","9"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0)");
+ }
+
+ @Test
+ public void quantile5bTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(quantileTest,
+ "QUANTILES='0.0','0.111','0.222','0.333','0.444','0.555','0.666','0.777','0.888','1.0'");
+
+ String[] input = {"1","2","3","4","10","5","6","7","8","9"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0)");
+ }
+
+ /**
+
+
+ define Median datafu.pig.stats.Median();
+
+ data_in = LOAD 'input' as (val:int);
+
+ --describe data_in;
+
+ data_out = GROUP data_in ALL;
+
+ --describe data_out;
+
+ data_out = FOREACH data_out {
+ sorted = ORDER data_in BY val;
+ GENERATE Median(sorted) as medians;
+ }
+ data_out = FOREACH data_out GENERATE FLATTEN(medians);
+
+ --describe data_out;
+
+ STORE data_out into 'output';
+ */
+ @Multiline private String medianTest;
+
+ @Test
+ public void medianTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(medianTest);
+
+ String[] input = {"4","5","6","9","10","7","8","2","3","1"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(5.5)");
+ }
+
+ /**
+
+
+ define Median datafu.pig.stats.StreamingMedian();
+
+ data_in = LOAD 'input' as (val:int);
+
+ --describe data_in;
+
+ data_out = GROUP data_in ALL;
+
+ --describe data_out;
+
+ data_out = FOREACH data_out {
+ GENERATE Median(data_in) as medians;
+ }
+ data_out = FOREACH data_out GENERATE FLATTEN(medians);
+
+ --describe data_out;
+
+ STORE data_out into 'output';
+ */
+ @Multiline private String streamingMedianTest;
+
+ @Test
+ public void streamingMedianTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(streamingMedianTest);
+
+ String[] input = {"0","4","5","6","9","10","7","8","2","3","1"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(5.0)");
+ }
+
+ /**
+
+
+ define Quantile datafu.pig.stats.StreamingQuantile($QUANTILES);
+
+ data_in = LOAD 'input' as (val:int);
+
+ --describe data_in;
+
+ data_out = GROUP data_in ALL;
+
+ --describe data_out;
+
+ data_out = FOREACH data_out GENERATE Quantile(data_in.val) as quantiles;
+ data_out = FOREACH data_out GENERATE FLATTEN(quantiles);
+
+ --describe data_out;
+
+ STORE data_out into 'output';
+ */
+ @Multiline private String streamingQuantileTest;
+
+ @Test
+ public void streamingQuantileTest() throws Exception {
+ PigTest test = createPigTestFromString(streamingQuantileTest,
+ "QUANTILES='5'");
+
+ String[] input = {"1","2","3","4","10","5","6","7","8","9"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(1.0,3.0,5.0,8.0,10.0)");
+ }
+
+ @Test
+ public void streamingQuantile2Test() throws Exception {
+ PigTest test = createPigTestFromString(streamingQuantileTest,
+ "QUANTILES='0.5','0.75','1.0'");
+
+ String[] input = {"1","2","3","4","10","5","6","7","8","9"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(5.0,8.0,10.0)");
+ }
+
+ @Test
+ public void streamingQuantile3Test() throws Exception {
+ PigTest test = createPigTestFromString(streamingQuantileTest,
+ "QUANTILES='0.07','0.03','0.37','1.0','0.0'");
+
+ List<String> input = new ArrayList<String>();
+ for (int i=1000; i>=1; i--)
+ {
+ input.add(Integer.toString(i));
+ }
+
+ writeLinesToFile("input", input.toArray(new String[0]));
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(70.0,30.0,370.0,1000.0,1.0)");
+ }
+
+ @Test
+ public void streamingQuantile4Test() throws Exception {
+ PigTest test = createPigTestFromString(streamingQuantileTest,
+ "QUANTILES='0.0013','0.0228','0.1587','0.5','0.8413','0.9772','0.9987'");
+
+ List<String> input = new ArrayList<String>();
+ for (int i=100000; i>=0; i--)
+ {
+ input.add(Integer.toString(i));
+ }
+
+ writeLinesToFile("input", input.toArray(new String[0]));
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(130.0,2280.0,15870.0,50000.0,84130.0,97720.0,99870.0)");
+ }
+
+ @Test
+ public void streamingQuantile5aTest() throws Exception {
+ PigTest test = createPigTestFromString(streamingQuantileTest,
+ "QUANTILES='10'");
+
+ String[] input = {"1","2","3","4","10","5","6","7","8","9"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0)");
+ }
+
+ @Test
+ public void streamingQuantile5bTest() throws Exception {
+ PigTest test = createPigTestFromString(streamingQuantileTest,
+ "QUANTILES='0.0','0.111','0.222','0.333','0.444','0.555','0.666','0.777','0.888','1.0'");
+
+ String[] input = {"1","2","3","4","10","5","6","7","8","9"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0)");
+ }
+
+ @Test
+ public void streamingQuantile6Test() throws Exception {
+ PigTest test = createPigTestFromString(streamingQuantileTest,
+ "QUANTILES='0.0','0.333','0.666','1.0'");
+
+ String[] input = {"1","2","3","4","10","5","6","7","8","9"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(1.0,4.0,7.0,10.0)");
+ }
+
+ @Test
+ public void streamingQuantile7Test() throws Exception {
+ PigTest test = createPigTestFromString(streamingQuantileTest,
+ "QUANTILES='4'");
+
+ String[] input = {"1","2","3","4","10","5","6","7","8","9"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(1.0,4.0,7.0,10.0)");
+ }
+
+ @Test
+ public void streamingQuantileExecTest() throws Exception {
+ StreamingQuantile quantile = new StreamingQuantile("4");
+
+ DataBag bag;
+ Tuple input;
+ Tuple result;
+
+ bag = BagFactory.getInstance().newDefaultBag();
+ for (int i=1; i<=10; i++)
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, i);
+ bag.add(t);
+ }
+ input = TupleFactory.getInstance().newTuple(bag);
+ result = quantile.exec(input);
+ Assert.assertEquals(4, result.size());
+ Assert.assertEquals(1.0, result.get(0));
+ Assert.assertEquals(4.0, result.get(1));
+ Assert.assertEquals(7.0, result.get(2));
+ Assert.assertEquals(10.0, result.get(3));
+
+ // do twice to check cleanup works
+
+ bag = BagFactory.getInstance().newDefaultBag();
+ for (int i=11; i<=20; i++)
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, i);
+ bag.add(t);
+ }
+ input = TupleFactory.getInstance().newTuple(bag);
+ result = quantile.exec(input);
+ Assert.assertEquals(4, result.size());
+ Assert.assertEquals(11.0, result.get(0));
+ Assert.assertEquals(14.0, result.get(1));
+ Assert.assertEquals(17.0, result.get(2));
+ Assert.assertEquals(20.0, result.get(3));
+ }
+
+ @Test
+ public void streamingQuantileAccumulateTest() throws Exception {
+ StreamingQuantile quantile = new StreamingQuantile("4");
+
+ Tuple result;
+
+ for (int i=1; i<=10; i++)
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, i);
+ DataBag bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(t);
+ Tuple input = TupleFactory.getInstance().newTuple(bag);
+ quantile.accumulate(input);
+ }
+ result = quantile.getValue();
+ Assert.assertEquals(4, result.size());
+ Assert.assertEquals(1.0, result.get(0));
+ Assert.assertEquals(4.0, result.get(1));
+ Assert.assertEquals(7.0, result.get(2));
+ Assert.assertEquals(10.0, result.get(3));
+
+ // do twice to check cleanup works
+ quantile.cleanup();
+
+ for (int i=11; i<=20; i++)
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, i);
+ DataBag bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(t);
+ Tuple input = TupleFactory.getInstance().newTuple(bag);
+ quantile.accumulate(input);
+ }
+ result = quantile.getValue();
+ Assert.assertEquals(4, result.size());
+ Assert.assertEquals(11.0, result.get(0));
+ Assert.assertEquals(14.0, result.get(1));
+ Assert.assertEquals(17.0, result.get(2));
+ Assert.assertEquals(20.0, result.get(3));
+ }
+
+ @Test
+ public void quantileParamsTest() throws Exception {
+ List<Double> quantiles = QuantileUtil.getQuantilesFromParams("5");
+
+ assertEquals(quantiles.size(),5);
+ assertAboutEqual(quantiles.get(0), 0.0);
+ assertAboutEqual(quantiles.get(1), 0.25);
+ assertAboutEqual(quantiles.get(2), 0.5);
+ assertAboutEqual(quantiles.get(3), 0.75);
+ assertAboutEqual(quantiles.get(4), 1.0);
+ }
+
+ @Test
+ public void quantileParamsTest2() throws Exception {
+ List<Double> quantiles = QuantileUtil.getQuantilesFromParams("2");
+
+ assertEquals(quantiles.size(),2);
+ assertAboutEqual(quantiles.get(0), 0.0);
+ assertAboutEqual(quantiles.get(1), 1.0);
+ }
+
+ @Test
+ public void quantileParamsTest3() throws Exception {
+ List<Double> quantiles = QuantileUtil.getQuantilesFromParams("11");
+
+ assertEquals(quantiles.size(),11);
+ assertAboutEqual(quantiles.get(0), 0.0);
+ assertAboutEqual(quantiles.get(1), 0.1);
+ assertAboutEqual(quantiles.get(2), 0.2);
+ assertAboutEqual(quantiles.get(3), 0.3);
+ assertAboutEqual(quantiles.get(4), 0.4);
+ assertAboutEqual(quantiles.get(5), 0.5);
+ assertAboutEqual(quantiles.get(6), 0.6);
+ assertAboutEqual(quantiles.get(7), 0.7);
+ assertAboutEqual(quantiles.get(8), 0.8);
+ assertAboutEqual(quantiles.get(9), 0.9);
+ assertAboutEqual(quantiles.get(10), 1.0);
+ }
+
+ @Test
+ public void quantileParamsTest4() throws Exception {
+ List<Double> quantiles = QuantileUtil.getQuantilesFromParams("10");
+
+ assertEquals(quantiles.size(),10);
+ assertAboutEqual(quantiles.get(0), 0.0);
+ assertAboutEqual(quantiles.get(1), 0.11111);
+ assertAboutEqual(quantiles.get(2), 0.22222);
+ assertAboutEqual(quantiles.get(3), 0.33333);
+ assertAboutEqual(quantiles.get(4), 0.44444);
+ assertAboutEqual(quantiles.get(5), 0.55555);
+ assertAboutEqual(quantiles.get(6), 0.66666);
+ assertAboutEqual(quantiles.get(7), 0.77777);
+ assertAboutEqual(quantiles.get(8), 0.88888);
+ assertAboutEqual(quantiles.get(9), 1.0);
+ }
+
+ @Test
+ public void quantileParamsTest5() throws Exception {
+ List<Double> quantiles = QuantileUtil.getQuantilesFromParams("4");
+
+ assertEquals(quantiles.size(),4);
+ assertAboutEqual(quantiles.get(0), 0.0);
+ assertAboutEqual(quantiles.get(1), 0.333);
+ assertAboutEqual(quantiles.get(2), 0.666);
+ assertAboutEqual(quantiles.get(3), 1.0);
+ }
+
+ private void assertAboutEqual(double actual, double expected) {
+ assertTrue(Math.abs(actual-expected) < 0.001);
+ }
+
+ /**
+
+
+ define Quantile datafu.pig.stats.$UDF($QUANTILES);
+
+ data_in = LOAD 'input' as (val:int);
+
+ --describe data_in;
+
+ data_out = GROUP data_in ALL;
+
+ --describe data_out;
+
+ data_out = FOREACH data_out {
+ sorted = ORDER data_in BY val;
+ GENERATE Quantile(sorted) as quantiles;
+ }
+ data_out = FOREACH data_out GENERATE FLATTEN(quantiles);
+
+ --describe data_out;
+
+ data_out = FOREACH data_out GENERATE $EXPECTED_OUTPUT;
+
+ STORE data_out into 'output';
+ */
+ @Multiline private String quantileSchemaTest;
+
+ @Test
+ public void quantileSchemaTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(quantileSchemaTest,
+ "UDF=Quantile",
+ "QUANTILES='0.0','0.5','1.0'",
+ "EXPECTED_OUTPUT=quantiles::quantile_0_0, "+
+ "quantiles::quantile_0_5, "+
+ "quantiles::quantile_1_0");
+
+ String[] input = {"1","5","3","4","2"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(1.0,3.0,5.0)");
+ }
+
+ @Test
+ public void quantileSchemaTest2() throws Exception
+ {
+ PigTest test = createPigTestFromString(quantileSchemaTest,
+ "UDF=Quantile",
+ "QUANTILES='3'",
+ "EXPECTED_OUTPUT=quantiles::quantile_0, "+
+ "quantiles::quantile_1, "+
+ "quantiles::quantile_2");
+
+ String[] input = {"1","5","3","4","2"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(1.0,3.0,5.0)");
+ }
+
+ @Test
+ public void quantileSchemaTest3() throws Exception
+ {
+ PigTest test = createPigTestFromString(quantileSchemaTest,
+ "UDF=StreamingQuantile",
+ "QUANTILES='0.0','0.5','1.0'",
+ "EXPECTED_OUTPUT=quantiles::quantile_0_0, "+
+ "quantiles::quantile_0_5, "+
+ "quantiles::quantile_1_0");
+
+ String[] input = {"1","5","3","4","2"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(1.0,3.0,5.0)");
+ }
+
+ @Test
+ public void quantileSchemaTest4() throws Exception
+ {
+ PigTest test = createPigTestFromString(quantileSchemaTest,
+ "UDF=StreamingQuantile",
+ "QUANTILES='3'",
+ "EXPECTED_OUTPUT=quantiles::quantile_0, "+
+ "quantiles::quantile_1, "+
+ "quantiles::quantile_2");
+
+ String[] input = {"1","5","3","4","2"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(1.0,3.0,5.0)");
+ }
+}
[18/19] DATAFU-27 Migrate build system to Gradle
Posted by wv...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/bags/EmptyBagToNullFields.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/bags/EmptyBagToNullFields.java b/datafu-pig/src/main/java/datafu/pig/bags/EmptyBagToNullFields.java
new file mode 100644
index 0000000..6933c28
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/bags/EmptyBagToNullFields.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.bags;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+import datafu.pig.util.ContextualEvalFunc;
+
+/**
+ * For an empty bag, inserts a tuple having null values for all fields;
+ * otherwise, the input bag is returned unchanged.
+ *
+ * <p>
+ * This can be useful when performing FLATTEN on a bag from a COGROUP,
+ * as FLATTEN on an empty bag produces no data.
+ * </p>
+ */
+public class EmptyBagToNullFields extends ContextualEvalFunc<DataBag>
+{
+ @Override
+ public DataBag exec(Tuple tuple) throws IOException
+ {
+ if (tuple.size() == 0 || tuple.get(0) == null)
+ return null;
+ Object o = tuple.get(0);
+ if (o instanceof DataBag)
+ {
+ DataBag bag = (DataBag)o;
+ if (bag.size() == 0)
+ {
+ // create a tuple with null values for all fields
+ int tupleSize = (Integer)getInstanceProperties().get("tuplesize");
+ return BagFactory.getInstance().newDefaultBag(Arrays.asList(TupleFactory.getInstance().newTuple(tupleSize)));
+ }
+ else
+ {
+ return bag;
+ }
+ }
+ else
+ throw new IllegalArgumentException("expected a null or a bag");
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try
+ {
+ if (input.size() != 1)
+ {
+ throw new RuntimeException("Expected only a single field as input");
+ }
+
+ if (input.getField(0).type != DataType.BAG)
+ {
+ throw new RuntimeException("Expected a BAG as input, but found " + DataType.findTypeName(input.getField(0).type));
+ }
+
+ // get the size of the tuple within the bag
+ int innerTupleSize = input.getField(0).schema.getField(0).schema.getFields().size();
+ getInstanceProperties().put("tuplesize", innerTupleSize);
+ }
+ catch (FrontendException e)
+ {
+ throw new RuntimeException(e);
+ }
+ return input;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/bags/Enumerate.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/bags/Enumerate.java b/datafu-pig/src/main/java/datafu/pig/bags/Enumerate.java
new file mode 100644
index 0000000..8a0d072
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/bags/Enumerate.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.bags;
+
+import java.io.IOException;
+
+import org.apache.pig.AccumulatorEvalFunc;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * Enumerate a bag, appending to each tuple its index within the bag.
+ *
+ * <p>
+ * For example:
+ * <pre>
+ * {(A),(B),(C),(D)} => {(A,0),(B,1),(C,2),(D,3)}
+ * </pre>
+ * The first constructor parameter (optional) dictates the starting index of the counting.
+ * This UDF implements the accumulator interface, reducing DataBag materialization costs.
+ * </p>
+ *
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * define Enumerate datafu.pig.bags.Enumerate('1');
+ *
+ * -- input:
+ * -- ({(100),(200),(300),(400)})
+ * input = LOAD 'input' as (B: bag{T: tuple(v2:INT)});
+ *
+ * -- output:
+ * -- ({(100,1),(200,2),(300,3),(400,4)})
+ * output = FOREACH input GENERATE Enumerate(B);
+ * }
+ * </pre>
+ */
+public class Enumerate extends AccumulatorEvalFunc<DataBag>
+{
+ private final int start;
+
+ private DataBag outputBag;
+ private long i;
+ private long count;
+
+ public Enumerate()
+ {
+ this("0");
+ }
+
+ public Enumerate(String start)
+ {
+ this.start = Integer.parseInt(start);
+ cleanup();
+ }
+
+ @Override
+ public void accumulate(Tuple arg0) throws IOException
+ {
+ DataBag inputBag = (DataBag)arg0.get(0);
+ for (Tuple t : inputBag) {
+ Tuple t1 = TupleFactory.getInstance().newTuple(t.getAll());
+ t1.append(i);
+ outputBag.add(t1);
+
+ if (count % 1000000 == 0) {
+ outputBag.spill();
+ count = 0;
+ }
+ i++;
+ count++;
+ }
+ }
+
+ @Override
+ public void cleanup()
+ {
+ this.outputBag = BagFactory.getInstance().newDefaultBag();
+ this.i = this.start;
+ this.count = 0;
+ }
+
+ @Override
+ public DataBag getValue()
+ {
+ return outputBag;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try {
+ if (input.size() != 1)
+ {
+ throw new RuntimeException("Expected input to have only a single field");
+ }
+
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.BAG)
+ {
+ throw new RuntimeException("Expected a BAG as input");
+ }
+
+ Schema inputBagSchema = inputFieldSchema.schema;
+
+ if (inputBagSchema.getField(0).type != DataType.TUPLE)
+ {
+ throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
+ DataType.findTypeName(inputBagSchema.getField(0).type)));
+ }
+
+ Schema inputTupleSchema = inputBagSchema.getField(0).schema;
+
+ Schema outputTupleSchema = inputTupleSchema.clone();
+ outputTupleSchema.add(new Schema.FieldSchema("i", DataType.LONG));
+
+ return new Schema(new Schema.FieldSchema(
+ getSchemaName(this.getClass().getName().toLowerCase(), input),
+ outputTupleSchema,
+ DataType.BAG));
+ }
+ catch (CloneNotSupportedException e) {
+ throw new RuntimeException(e);
+ }
+ catch (FrontendException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/bags/FirstTupleFromBag.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/bags/FirstTupleFromBag.java b/datafu-pig/src/main/java/datafu/pig/bags/FirstTupleFromBag.java
new file mode 100644
index 0000000..1f24984
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/bags/FirstTupleFromBag.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.bags;
+
+import java.io.IOException;
+
+import datafu.pig.util.SimpleEvalFunc;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * Returns the first tuple from a bag. Requires a second parameter that will be returned if the bag is empty.
+ *
+ * Example:
+ * <pre>
+ * {@code
+ * define FirstTupleFromBag datafu.pig.bags.FirstTupleFromBag();
+ *
+ * -- input:
+ * -- ({(a,1)})
+ * input = LOAD 'input' AS (B: bag {T: tuple(alpha:CHARARRAY, numeric:INT)});
+ *
+ * output = FOREACH input GENERATE FirstTupleFromBag(B, null);
+ *
+ * -- output:
+ * -- (a,1)
+ * }
+ * </pre>
+ */
+
+public class FirstTupleFromBag extends SimpleEvalFunc<Tuple>
+{
+ public Tuple call(DataBag bag, Tuple defaultValue) throws IOException
+ {
+ for (Tuple t : bag) {
+ return t;
+ }
+ return defaultValue;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try {
+ return new Schema(input.getField(0).schema);
+ }
+ catch (Exception e) {
+ return null;
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/bags/NullToEmptyBag.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/bags/NullToEmptyBag.java b/datafu-pig/src/main/java/datafu/pig/bags/NullToEmptyBag.java
new file mode 100644
index 0000000..09fffb3
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/bags/NullToEmptyBag.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.bags;
+
+import java.io.IOException;
+
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * Returns an empty bag if the input is null; otherwise,
+ * returns the input bag unchanged.
+ */
+public class NullToEmptyBag extends EvalFunc<DataBag>
+{
+ @Override
+ public DataBag exec(Tuple tuple) throws IOException
+ {
+ if (tuple.size() == 0 || tuple.get(0) == null)
+ return BagFactory.getInstance().newDefaultBag();
+ Object o = tuple.get(0);
+ if (o instanceof DataBag)
+ return (DataBag)o;
+ else
+ throw new IllegalArgumentException("expected a null or a bag");
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ return input;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/bags/PrependToBag.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/bags/PrependToBag.java b/datafu-pig/src/main/java/datafu/pig/bags/PrependToBag.java
new file mode 100644
index 0000000..9292871
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/bags/PrependToBag.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.bags;
+
+import java.io.IOException;
+
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+import datafu.pig.util.SimpleEvalFunc;
+
+/**
+ * Prepends a tuple to a bag.
+ *
+ * <p>N.B. this copies the entire input bag, so don't use it for large bags.</p>
+ *
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * define PrependToBag datafu.pig.bags.PrependToBag();
+ *
+ * -- input:
+ * -- ({(1),(2),(3)},(4))
+ * -- ({(10),(20),(30),(40),(50)},(60))
+ * input = LOAD 'input' AS (B: bag{T: tuple(v:INT)}, T: tuple(v:INT));
+
+ * -- output:
+ * -- ({(4),(1),(2),(3)})
+ * -- ({(60),(10),(20),(30),(40),(50)})
+ * output = FOREACH input GENERATE PrependToBag(B,T) as B;
+ * }
+ * </pre>
+ * </p>
+ */
+public class PrependToBag extends SimpleEvalFunc<DataBag>
+{
+ public DataBag call(DataBag inputBag, Tuple t) throws IOException
+ {
+ DataBag outputBag = BagFactory.getInstance().newDefaultBag();
+ outputBag.add(t);
+ for (Tuple x : inputBag)
+ outputBag.add(x);
+ return outputBag;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try {
+ return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
+ input.getField(0).schema, DataType.BAG));
+ }
+ catch (FrontendException e) {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/bags/ReverseEnumerate.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/bags/ReverseEnumerate.java b/datafu-pig/src/main/java/datafu/pig/bags/ReverseEnumerate.java
new file mode 100644
index 0000000..c86ffcf
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/bags/ReverseEnumerate.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.bags;
+
+import java.io.IOException;
+
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+import datafu.pig.util.SimpleEvalFunc;
+
+/**
+ * Enumerate a bag, appending to each tuple its index within the bag, with indices being produced in
+ * descending order.
+ *
+ * <p>
+ * For example:
+ * <pre>
+ * {(A),(B),(C),(D)} => {(A,3),(B,2),(C,1),(D,0)}
+ * </pre>
+ * The first constructor parameter (optional) dictates the starting index of the counting. As the
+ * UDF requires the size of the bag for reverse counting, this UDF does <b>not</b> implement the
+ * accumulator interface and suffers from the slight performance penalty of DataBag materialization.
+ * </p>
+ *
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * define ReverseEnumerate datafu.pig.bags.ReverseEnumerate('1');
+ *
+ * -- input:
+ * -- ({(100),(200),(300),(400)})
+ * input = LOAD 'input' as (B: bag{T: tuple(v2:INT)});
+ *
+ * -- output:
+ * -- ({(100,4),(200,3),(300,2),(400,1)})
+ * output = FOREACH input GENERATE ReverseEnumerate(B);
+ * }
+ * </pre>
+ * </p>
+ */
+public class ReverseEnumerate extends SimpleEvalFunc<DataBag>
+{
+ private final int start;
+
+ public ReverseEnumerate()
+ {
+ this.start = 0;
+ }
+
+ public ReverseEnumerate(String start)
+ {
+ this.start = Integer.parseInt(start);
+ }
+
+ public DataBag call(DataBag inputBag) throws IOException
+ {
+ DataBag outputBag = BagFactory.getInstance().newDefaultBag();
+ long i = start, count = 0;
+ i = inputBag.size() - 1 + start;
+
+ for (Tuple t : inputBag) {
+ Tuple t1 = TupleFactory.getInstance().newTuple(t.getAll());
+ t1.append(i);
+ outputBag.add(t1);
+
+ if (count % 1000000 == 0) {
+ outputBag.spill();
+ count = 0;
+ }
+ i--;
+ count++;
+ }
+
+ return outputBag;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try {
+ if (input.size() != 1)
+ {
+ throw new RuntimeException("Expected input to have only a single field");
+ }
+
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.BAG)
+ {
+ throw new RuntimeException("Expected a BAG as input");
+ }
+
+ Schema inputBagSchema = inputFieldSchema.schema;
+
+ if (inputBagSchema.getField(0).type != DataType.TUPLE)
+ {
+ throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
+ DataType.findTypeName(inputBagSchema.getField(0).type)));
+ }
+
+ Schema inputTupleSchema = inputBagSchema.getField(0).schema;
+
+ Schema outputTupleSchema = inputTupleSchema.clone();
+ outputTupleSchema.add(new Schema.FieldSchema("i", DataType.LONG));
+
+ return new Schema(new Schema.FieldSchema(
+ getSchemaName(this.getClass().getName().toLowerCase(), input),
+ outputTupleSchema,
+ DataType.BAG));
+ }
+ catch (CloneNotSupportedException e) {
+ throw new RuntimeException(e);
+ }
+ catch (FrontendException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/bags/UnorderedPairs.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/bags/UnorderedPairs.java b/datafu-pig/src/main/java/datafu/pig/bags/UnorderedPairs.java
new file mode 100644
index 0000000..a1d149e
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/bags/UnorderedPairs.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.bags;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.tools.pigstats.PigStatusReporter;
+
+/**
+ * Generates pairs of all items in a bag.
+ *
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * define UnorderedPairs datafu.pig.bags.UnorderedPairs();
+ *
+ * -- input:
+ * -- ({(1),(2),(3),(4)})
+ * input = LOAD 'input' AS (B: bag {T: tuple(v:INT)});
+ *
+ * -- output:
+ * -- ({((1),(2)),((1),(3)),((1),(4)),((2),(3)),((2),(4)),((3),(4))})
+ * output = FOREACH input GENERATE UnorderedPairs(B);
+ * }
+ * </pre>
+ * </p>
+ */
+public class UnorderedPairs extends EvalFunc<DataBag>
+{
+ private static final BagFactory bagFactory = BagFactory.getInstance();
+ private static final TupleFactory tupleFactory = TupleFactory.getInstance();
+
+ @Override
+ public DataBag exec(Tuple input) throws IOException
+ {
+ PigStatusReporter reporter = PigStatusReporter.getInstance();
+
+ try {
+ DataBag inputBag = (DataBag) input.get(0);
+ DataBag outputBag = bagFactory.newDefaultBag();
+ long i=0, j, cnt=0;
+
+ if (inputBag != null)
+ {
+ for (Tuple elem1 : inputBag) {
+ j = 0;
+ for (Tuple elem2 : inputBag) {
+ if (j > i) {
+ outputBag.add(tupleFactory.newTuple(Arrays.asList(elem1, elem2)));
+ cnt++;
+ }
+ j++;
+
+ if (reporter != null)
+ reporter.progress();
+
+ if (cnt % 1000000 == 0) {
+ outputBag.spill();
+ cnt = 0;
+ }
+ }
+ i++;
+ }
+ }
+
+ return outputBag;
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Caught exception processing input of " + this.getClass().getName(), e);
+ }
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try {
+ if (input.size() != 1)
+ {
+ throw new RuntimeException("Expected input to have only a single field");
+ }
+
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.BAG)
+ {
+ throw new RuntimeException("Expected a BAG as input");
+ }
+
+ Schema inputBagSchema = inputFieldSchema.schema;
+
+ if (inputBagSchema.getField(0).type != DataType.TUPLE)
+ {
+ throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
+ DataType.findTypeName(inputBagSchema.getField(0).type)));
+ }
+
+ Schema ouputTupleSchema = new Schema();
+ ouputTupleSchema.add(new Schema.FieldSchema("elem1", inputBagSchema.getField(0).schema.clone(), DataType.TUPLE));
+ ouputTupleSchema.add(new Schema.FieldSchema("elem2", inputBagSchema.getField(0).schema.clone(), DataType.TUPLE));
+ return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
+ ouputTupleSchema,
+ DataType.BAG));
+ }
+ catch (Exception e) {
+ return null;
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/bags/package-info.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/bags/package-info.java b/datafu-pig/src/main/java/datafu/pig/bags/package-info.java
new file mode 100644
index 0000000..214b837
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/bags/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * A collection of general purpose UDFs for operating on bags.
+ */
+package datafu.pig.bags;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/geo/HaversineDistInMiles.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/geo/HaversineDistInMiles.java b/datafu-pig/src/main/java/datafu/pig/geo/HaversineDistInMiles.java
new file mode 100644
index 0000000..d1e3988
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/geo/HaversineDistInMiles.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.geo;
+
+import org.apache.pig.data.DataType;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+import datafu.pig.util.SimpleEvalFunc;
+
+/**
+ * Computes the distance (in miles) between two latitude-longitude pairs
+ * using the {@link <a href="http://en.wikipedia.org/wiki/Haversine_formula" target="_blank">Haversine formula</a>}.
+ *
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * -- input is a TSV of two latitude and longitude pairs
+ * input = LOAD 'input' AS (lat1 : double, long1 : double, lat2 : double, long2 : double);
+ * output = FOREACH input GENERATE datafu.pig.geo.HaversineDistInMiles(lat1, long1, lat2, long2) as distance;
+ * }</pre></p>
+ */
+public class HaversineDistInMiles extends SimpleEvalFunc<Double>
+{
+ public static final double EARTH_RADIUS = 3958.75;
+
+ public Double call(Double lat1, Double lng1, Double lat2, Double lng2)
+ {
+ if (lat1 == null || lng1 == null || lat2 == null || lng2 == null)
+ return null;
+
+ double d_lat = Math.toRadians(lat2-lat1);
+ double d_long = Math.toRadians(lng2-lng1);
+ double a = Math.sin(d_lat/2) * Math.sin(d_lat/2) +
+ Math.cos(Math.toRadians(lat1)) * Math.cos(Math.toRadians(lat2)) *
+ Math.sin(d_long/2) * Math.sin(d_long/2);
+ double c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1-a));
+ return EARTH_RADIUS * c;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ return new Schema(new Schema.FieldSchema("dist", DataType.DOUBLE));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/geo/package-info.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/geo/package-info.java b/datafu-pig/src/main/java/datafu/pig/geo/package-info.java
new file mode 100644
index 0000000..12b27b0
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/geo/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * UDFs for geographic computations.
+ */
+package datafu.pig.geo;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/hash/MD5.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/hash/MD5.java b/datafu-pig/src/main/java/datafu/pig/hash/MD5.java
new file mode 100644
index 0000000..b7b51da
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/hash/MD5.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.hash;
+
+import java.math.BigInteger;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+import org.apache.commons.codec.binary.Base64;
+
+import datafu.pig.util.SimpleEvalFunc;
+
+/**
+ * Computes the MD5 value of a string and outputs it in hex (by default).
+ * A method can be provided to the constructor, which may be either 'hex' or 'base64'.
+ */
+public class MD5 extends SimpleEvalFunc<String>
+{
+ private final MessageDigest md5er;
+ private final boolean isBase64;
+
+ public MD5()
+ {
+ this("hex");
+ }
+
+ public MD5(String method)
+ {
+ if ("hex".equals(method))
+ {
+ isBase64 = false;
+ }
+ else if ("base64".equals(method))
+ {
+ isBase64 = true;
+ }
+ else
+ {
+ throw new IllegalArgumentException("Expected either hex or base64");
+ }
+
+ try {
+ md5er = MessageDigest.getInstance("md5");
+ }
+ catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public String call(String val)
+ {
+ if (isBase64)
+ {
+ return new String(Base64.encodeBase64(md5er.digest(val.getBytes())));
+ }
+ else
+ {
+ return new BigInteger(1, md5er.digest(val.getBytes())).toString(16);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/hash/SHA.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/hash/SHA.java b/datafu-pig/src/main/java/datafu/pig/hash/SHA.java
new file mode 100644
index 0000000..ff859e5
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/hash/SHA.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.hash;
+
+import java.math.BigInteger;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+import datafu.pig.util.SimpleEvalFunc;
+
+public class SHA extends SimpleEvalFunc<String> {
+ private final MessageDigest sha;
+
+ public SHA(){
+ this("256");
+ }
+
+ public SHA(String algorithm){
+ try {
+ sha = MessageDigest.getInstance("SHA-"+algorithm);
+ } catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public String call(String value){
+ return new BigInteger(1, sha.digest(value.getBytes())).toString(16);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/hash/package-info.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/hash/package-info.java b/datafu-pig/src/main/java/datafu/pig/hash/package-info.java
new file mode 100644
index 0000000..320a029
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/hash/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * UDFs for computing hashes from data.
+ */
+package datafu.pig.hash;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/linkanalysis/PageRank.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/linkanalysis/PageRank.java b/datafu-pig/src/main/java/datafu/pig/linkanalysis/PageRank.java
new file mode 100644
index 0000000..80ff567
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/linkanalysis/PageRank.java
@@ -0,0 +1,482 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.linkanalysis;
+
+import it.unimi.dsi.fastutil.ints.Int2IntMap;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.pig.Accumulator;
+import org.apache.pig.AccumulatorEvalFunc;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * A UDF which implements {@link <a href="http://en.wikipedia.org/wiki/PageRank" target="_blank">PageRank</a>}.
+ *
+ * <p>
+ * This is not a distributed implementation. Each graph is stored in memory while running the algorithm, with edges optionally
+ * spilled to disk to conserve memory. This can be used to distribute the execution of PageRank on multiple
+ * reasonably sized graphs. It does not distribute execuion of PageRank for each individual graph. Each graph is identified
+ * by an integer valued topic ID.
+ * </p>
+ *
+ * <p>
+ * If the graph is too large to fit in memory than an alternative method must be used, such as an iterative approach which runs
+ * many MapReduce jobs in a sequence to complete the PageRank iterations.
+ * </p>
+ *
+ * <p>
+ * Each graph is represented through a bag of (source,edges) tuples. The 'source' is an integer ID representing the source node.
+ * The 'edges' are the outgoing edges from the source node, represented as a bag of (dest,weight) tuples. The 'dest' is an
+ * integer ID representing the destination node. The weight is a double representing how much the edge should be weighted.
+ * For a standard PageRank implementation just use weight of 1.0.
+ * </p>
+ *
+ * <p>
+ * The output of the UDF is a bag of (source,rank) pairs, where 'rank' is the PageRank value for that source in the graph.
+ * </p>
+ *
+ * <p>
+ * There are several configurable options for this UDF, among them:
+ * <p>
+ *
+ * <ul>
+ * <li>
+ * <b>alpha</b>: Controls the PageRank alpha value. The default is 0.85. A higher value reduces the "random jump"
+ * factor and causes the rank to be influenced more by edges.
+ * </li>
+ * <li>
+ * <b>max_iters</b>: The maximum number of iterations to run. The default is 150.
+ * </li>
+ * <li>
+ * <b>dangling_nodes</b>: How to handling "dangling nodes", i.e. nodes with no outgoing edges. When "true" this is equivalent
+ * to forcing a dangling node to have an outgoing edge to every other node in the graph. The default is "false".
+ * </li>
+ * <li>
+ * <b>tolerance</b>: A threshold which causes iterations to cease. It is measured from the total change in ranks from each of
+ * the nodes in the graph. As the ranks settle on their final values the total change decreases. This can be used
+ * to stop iterations early. The default is 1e-16.
+ * </li>
+ * <li>
+ * <b>max_nodes_and_edges</b>: This is a control to prevent running out of memory. As a graph is loaded, if the sum of edges
+ * and nodes exceeds this value then it will stop. It will not fail but PageRank will not be run on this graph. Instead a null
+ * value will be returned as a result. The default is 100M.
+ * </li>
+ * <li>
+ * <b>spill_to_edge_disk_storage</b>: Used to conserve memory. When "true" it causes the edge data to be written to disk in a temp file instead
+ * of being held in memory when the number of edges exceeds a threshold. The nodes are still held in memory however.
+ * Each iteration of PageRank will stream through the edges stored on disk. The default is "false".
+ * </li>
+ * <li>
+ * <b>max_edges_in_memory</b>: When spilling edges to disk is enabled, this is the threshold which triggers that behavior. The default is 30M.
+ * </li>
+ * </ul>
+ *
+ * <p>
+ * Parameters are configured by passing them in as a sequence of pairs into the UDF constructor. For example, below the alpha value is set to
+ * 0.87 and dangling nodes are enabled. All arguments must be strings.
+ * </p>
+ *
+ * <p>
+ * <pre>
+ * {@code
+ * define PageRank datafu.pig.linkanalysis.PageRank('alpha','0.87','dangling_nodes','true');
+ * }
+ * </pre>
+ * </p>
+ *
+ * <p>
+ * Full example:
+ * <pre>
+ * {@code
+ *
+ * topic_edges = LOAD 'input_edges' as (topic:INT,source:INT,dest:INT,weight:DOUBLE);
+ *
+ * topic_edges_grouped = GROUP topic_edges by (topic, source) ;
+ * topic_edges_grouped = FOREACH topic_edges_grouped GENERATE
+ * group.topic as topic,
+ * group.source as source,
+ * topic_edges.(dest,weight) as edges;
+ *
+ * topic_edges_grouped_by_topic = GROUP topic_edges_grouped BY topic;
+ *
+ * topic_ranks = FOREACH topic_edges_grouped_by_topic GENERATE
+ * group as topic,
+ * FLATTEN(PageRank(topic_edges_grouped.(source,edges))) as (source,rank);
+ *
+ * topic_ranks = FOREACH topic_ranks GENERATE
+ * topic, source, rank;
+ *
+ * }
+ * </pre>
+ * </p>
+ */
+public class PageRank extends AccumulatorEvalFunc<DataBag>
+{
+ private final datafu.pig.linkanalysis.PageRankImpl graph = new datafu.pig.linkanalysis.PageRankImpl();
+
+ private int maxNodesAndEdges = 100000000;
+ private int maxEdgesInMemory = 30000000;
+ private double tolerance = 1e-16;
+ private int maxIters = 150;
+ private boolean useEdgeDiskStorage = false;
+ private boolean enableDanglingNodeHandling = false;
+ private boolean enableNodeBiasing = false;
+ private boolean aborted = false;
+ private float alpha = 0.85f;
+
+ TupleFactory tupleFactory = TupleFactory.getInstance();
+ BagFactory bagFactory = BagFactory.getInstance();
+
+ public PageRank()
+ {
+ initialize();
+ }
+
+ public PageRank(String... parameters)
+ {
+ if (parameters.length % 2 != 0)
+ {
+ throw new RuntimeException("Invalid parameters list");
+ }
+
+ for (int i=0; i<parameters.length; i+=2)
+ {
+ String parameterName = parameters[i];
+ String value = parameters[i+1];
+ if (parameterName.equals("max_nodes_and_edges"))
+ {
+ maxNodesAndEdges = Integer.parseInt(value);
+ }
+ else if (parameterName.equals("max_edges_in_memory"))
+ {
+ maxEdgesInMemory = Integer.parseInt(value);
+ }
+ else if (parameterName.equals("tolerance"))
+ {
+ tolerance = Double.parseDouble(value);
+ }
+ else if (parameterName.equals("max_iters"))
+ {
+ maxIters = Integer.parseInt(value);
+ }
+ else if (parameterName.equals("spill_to_edge_disk_storage"))
+ {
+ useEdgeDiskStorage = Boolean.parseBoolean(value);
+ }
+ else if (parameterName.equals("dangling_nodes"))
+ {
+ enableDanglingNodeHandling = Boolean.parseBoolean(value);
+ }
+ else if (parameterName.equals("node_biasing"))
+ {
+ enableNodeBiasing = Boolean.parseBoolean(value);
+ }
+ else if (parameterName.equals("alpha"))
+ {
+ alpha = Float.parseFloat(value);
+ }
+ }
+
+ initialize();
+ }
+
+ private void initialize()
+ {
+ if (useEdgeDiskStorage)
+ {
+ this.graph.enableEdgeDiskCaching();
+ }
+ else
+ {
+ this.graph.disableEdgeDiskCaching();
+ }
+
+ if (enableDanglingNodeHandling)
+ {
+ this.graph.enableDanglingNodeHandling();
+ }
+ else
+ {
+ this.graph.disableDanglingNodeHandling();
+ }
+
+ if (enableNodeBiasing)
+ {
+ this.graph.enableNodeBiasing();
+ }
+ else
+ {
+ this.graph.disableNodeBiasing();
+ }
+
+ this.graph.setEdgeCachingThreshold(maxEdgesInMemory);
+ this.graph.setAlpha(alpha);
+ }
+
+ @Override
+ public void accumulate(Tuple t) throws IOException
+ {
+ if (aborted)
+ {
+ return;
+ }
+
+ DataBag bag = (DataBag) t.get(0);
+ if (bag == null || bag.size() == 0)
+ return;
+
+ for (Tuple sourceTuple : bag)
+ {
+ Integer sourceId = (Integer)sourceTuple.get(0);
+ DataBag edges = (DataBag)sourceTuple.get(1);
+ Double nodeBias = null;
+ if (enableNodeBiasing)
+ {
+ nodeBias = (Double)sourceTuple.get(2);
+ }
+
+ ArrayList<Map<String,Object>> edgesMapList = new ArrayList<Map<String, Object>>();
+
+ for (Tuple edgeTuple : edges)
+ {
+ Integer destId = (Integer)edgeTuple.get(0);
+ Double weight = (Double)edgeTuple.get(1);
+ HashMap<String,Object> edgeMap = new HashMap<String, Object>();
+ edgeMap.put("dest",destId);
+ edgeMap.put("weight",weight);
+ edgesMapList.add(edgeMap);
+ }
+
+ if (enableNodeBiasing)
+ {
+ graph.addNode(sourceId, edgesMapList, nodeBias.floatValue());
+ }
+ else
+ {
+ graph.addNode(sourceId, edgesMapList);
+ }
+
+ if (graph.nodeCount() + graph.edgeCount() > maxNodesAndEdges)
+ {
+ System.out.println(String.format("There are too many nodes and edges (%d + %d > %d). Aborting.", graph.nodeCount(), graph.edgeCount(), maxNodesAndEdges));
+ aborted = true;
+ break;
+ }
+
+ reporter.progress();
+ }
+ }
+
+ @Override
+ public DataBag getValue()
+ {
+ if (aborted)
+ {
+ return null;
+ }
+
+ System.out.println(String.format("Nodes: %d, Edges: %d", graph.nodeCount(), graph.edgeCount()));
+
+ ProgressIndicator progressIndicator = getProgressIndicator();
+ System.out.println("Finished loading graph.");
+ long startTime = System.nanoTime();
+ System.out.println("Initializing.");
+ try
+ {
+ graph.init(progressIndicator);
+ }
+ catch (IOException e)
+ {
+ e.printStackTrace();
+ return null;
+ }
+ System.out.println(String.format("Done, took %f ms", (System.nanoTime() - startTime)/10.0e6));
+
+ float totalDiff;
+ int iter = 0;
+
+ System.out.println("Beginning iterations");
+ startTime = System.nanoTime();
+ do
+ {
+ // TODO log percentage complete every 5 minutes
+ try
+ {
+ totalDiff = graph.nextIteration(progressIndicator);
+ }
+ catch (IOException e)
+ {
+ e.printStackTrace();
+ return null;
+ }
+ iter++;
+ } while(iter < maxIters && totalDiff > tolerance);
+ System.out.println(String.format("Done, %d iterations took %f ms", iter, (System.nanoTime() - startTime)/10.0e6));
+
+ DataBag output = bagFactory.newDefaultBag();
+
+ for (Int2IntMap.Entry node : graph.getNodeIds())
+ {
+ int nodeId = node.getIntKey();
+ float rank = graph.getNodeRank(nodeId);
+ List nodeData = new ArrayList(2);
+ nodeData.add(nodeId);
+ nodeData.add(rank);
+ output.add(tupleFactory.newTuple(nodeData));
+ }
+
+ return output;
+ }
+
+ @Override
+ public void cleanup()
+ {
+ try
+ {
+ aborted = false;
+ this.graph.clear();
+ }
+ catch (IOException e)
+ {
+ e.printStackTrace();
+ }
+ }
+
+ private ProgressIndicator getProgressIndicator()
+ {
+ return new ProgressIndicator()
+ {
+ @Override
+ public void progress()
+ {
+ reporter.progress();
+ }
+ };
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try
+ {
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.BAG)
+ {
+ throw new RuntimeException("Expected a BAG as input");
+ }
+
+ Schema inputBagSchema = inputFieldSchema.schema;
+
+ if (inputBagSchema.getField(0).type != DataType.TUPLE)
+ {
+ throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
+ DataType.findTypeName(inputBagSchema.getField(0).type)));
+ }
+
+ Schema inputTupleSchema = inputBagSchema.getField(0).schema;
+
+ if (!this.enableNodeBiasing)
+ {
+ if (inputTupleSchema.size() != 2)
+ {
+ throw new RuntimeException("Expected two fields for the node data");
+ }
+ }
+ else
+ {
+ if (inputTupleSchema.size() != 3)
+ {
+ throw new RuntimeException("Expected three fields for the node data");
+ }
+ }
+
+ if (inputTupleSchema.getField(0).type != DataType.INTEGER)
+ {
+ throw new RuntimeException(String.format("Expected source to be an INTEGER, but instead found %s",
+ DataType.findTypeName(inputTupleSchema.getField(0).type)));
+ }
+
+ if (inputTupleSchema.getField(1).type != DataType.BAG)
+ {
+ throw new RuntimeException(String.format("Expected edges to be represented with a BAG"));
+ }
+
+ if (this.enableNodeBiasing && inputTupleSchema.getField(2).type != DataType.DOUBLE)
+ {
+ throw new RuntimeException(String.format("Expected node bias to be a DOUBLE, but instead found %s",
+ DataType.findTypeName(inputTupleSchema.getField(2).type)));
+ }
+
+ Schema.FieldSchema edgesFieldSchema = inputTupleSchema.getField(1);
+
+ if (edgesFieldSchema.schema.getField(0).type != DataType.TUPLE)
+ {
+ throw new RuntimeException(String.format("Expected edges field to contain a TUPLE, but instead found %s",
+ DataType.findTypeName(edgesFieldSchema.schema.getField(0).type)));
+ }
+
+ Schema edgesTupleSchema = edgesFieldSchema.schema.getField(0).schema;
+
+ if (edgesTupleSchema.size() != 2)
+ {
+ throw new RuntimeException("Expected two fields for the edge data");
+ }
+
+ if (edgesTupleSchema.getField(0).type != DataType.INTEGER)
+ {
+ throw new RuntimeException(String.format("Expected destination edge ID to an INTEGER, but instead found %s",
+ DataType.findTypeName(edgesTupleSchema.getField(0).type)));
+ }
+
+ if (edgesTupleSchema.getField(1).type != DataType.DOUBLE)
+ {
+ throw new RuntimeException(String.format("Expected destination edge weight to a DOUBLE, but instead found %s",
+ DataType.findTypeName(edgesTupleSchema.getField(1).type)));
+ }
+
+ Schema tupleSchema = new Schema();
+ tupleSchema.add(new Schema.FieldSchema("node",DataType.INTEGER));
+ tupleSchema.add(new Schema.FieldSchema("rank",DataType.FLOAT));
+
+ return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
+ .getName()
+ .toLowerCase(), input),
+ tupleSchema,
+ DataType.BAG));
+ }
+ catch (FrontendException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/linkanalysis/PageRankImpl.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/linkanalysis/PageRankImpl.java b/datafu-pig/src/main/java/datafu/pig/linkanalysis/PageRankImpl.java
new file mode 100644
index 0000000..5d0b932
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/linkanalysis/PageRankImpl.java
@@ -0,0 +1,571 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.linkanalysis;
+
+import it.unimi.dsi.fastutil.floats.FloatArrayList;
+import it.unimi.dsi.fastutil.ints.Int2IntMap;
+import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Map;
+
+import com.google.common.collect.AbstractIterator;
+
+/**
+ * An implementation of {@link <a href="http://en.wikipedia.org/wiki/PageRank" target="_blank">PageRank</a>}, used by the {@link PageRank} UDF.
+ * It is not intended to be used directly.
+ * </p>
+ */
+public class PageRankImpl
+{
+ private float totalRankChange;
+ private long edgeCount;
+ private long nodeCount;
+
+ // the damping factor
+ private float alpha = 0.85f;
+
+ // edge weights (which are doubles) are multiplied by this value so they can be stored as integers internally
+ private static float EDGE_WEIGHT_MULTIPLIER = 100000;
+
+ private final Int2IntOpenHashMap nodeIndices = new Int2IntOpenHashMap();
+ private final FloatArrayList nodeData = new FloatArrayList(); // rank, total weight, contribution, bias(optional), (repeat)
+ private int nodeFieldCount = 3; // unless biasing is enabled
+
+ private final IntArrayList danglingNodes = new IntArrayList();
+
+ private final IntArrayList edges = new IntArrayList(); // source, dest node count... dest id, weight pos, (repeat)
+
+ private boolean shouldHandleDanglingNodes = false;
+ private boolean shouldCacheEdgesOnDisk = false;
+ private long edgeCachingThreshold;
+ private boolean nodeBiasingEnabled = false;
+
+ private File edgesFile;
+ private DataOutputStream edgeDataOutputStream;
+ private boolean usingEdgeDiskCache;
+
+ public void clear() throws IOException
+ {
+ this.edgeCount = 0;
+ this.nodeCount = 0;
+ this.totalRankChange = 0.0f;
+
+ this.nodeIndices.clear();
+ this.nodeData.clear();
+ this.edges.clear();
+ this.danglingNodes.clear();
+
+ if (edgeDataOutputStream != null)
+ {
+ this.edgeDataOutputStream.close();
+ this.edgeDataOutputStream = null;
+ }
+
+ this.usingEdgeDiskCache = false;
+ this.edgesFile = null;
+ }
+
+ /**
+ * Gets the page rank alpha value.
+ * @return alpha
+ */
+ public float getAlpha()
+ {
+ return alpha;
+ }
+
+ /**
+ * Sets the page rank alpha value (default is 0.85);
+ * @param alpha
+ */
+ public void setAlpha(float alpha)
+ {
+ this.alpha = alpha;
+ }
+
+ public boolean isNodeBiasingEnabled()
+ {
+ return this.nodeBiasingEnabled;
+ }
+
+ public void enableNodeBiasing()
+ {
+ this.nodeBiasingEnabled = true;
+ this.nodeFieldCount = 4;
+ }
+
+ public void disableNodeBiasing()
+ {
+ this.nodeBiasingEnabled = false;
+ this.nodeFieldCount = 3;
+ }
+
+
+ /**
+ * Gets whether disk is being used to cache edges.
+ * @return True if the edges are cached on disk.
+ */
+ public boolean isUsingEdgeDiskCache()
+ {
+ return usingEdgeDiskCache;
+ }
+
+ /**
+ * Enable disk caching of edges once there are too many (disabled by default).
+ */
+ public void enableEdgeDiskCaching()
+ {
+ shouldCacheEdgesOnDisk = true;
+ }
+
+ /**
+ * Disable disk caching of edges once there are too many (disabled by default).
+ */
+ public void disableEdgeDiskCaching()
+ {
+ shouldCacheEdgesOnDisk = false;
+ }
+
+ /**
+ * Gets whether edge disk caching is enabled.
+ * @return True if edge disk caching is enabled.
+ */
+ public boolean isEdgeDiskCachingEnabled()
+ {
+ return shouldCacheEdgesOnDisk;
+ }
+
+ /**
+ * Gets the number of edges past which they will be cached on disk instead of in memory.
+ * Edge disk caching must be enabled for this to have any effect.
+ * @return Edge count past which caching occurs
+ */
+ public long getEdgeCachingThreshold()
+ {
+ return edgeCachingThreshold;
+ }
+
+ /**
+ * Set the number of edges past which they will be cached on disk instead of in memory.
+ * Edge disk caching must be enabled for this to have any effect.
+ * @param count Edge count past which caching occurs
+ */
+ public void setEdgeCachingThreshold(long count)
+ {
+ edgeCachingThreshold = count;
+ }
+
+ /**
+ * Enables dangling node handling (disabled by default).
+ */
+ public void enableDanglingNodeHandling()
+ {
+ shouldHandleDanglingNodes = true;
+ }
+
+ /**
+ * Disables dangling node handling (disabled by default).
+ */
+ public void disableDanglingNodeHandling()
+ {
+ shouldHandleDanglingNodes = false;
+ }
+
+ public long nodeCount()
+ {
+ return this.nodeCount;
+ }
+
+ public long edgeCount()
+ {
+ return this.edgeCount;
+ }
+
+ public Int2IntMap.FastEntrySet getNodeIds()
+ {
+ return this.nodeIndices.int2IntEntrySet();
+ }
+
+ public float getNodeRank(int nodeId)
+ {
+ int nodeIndex = this.nodeIndices.get(nodeId);
+ return nodeData.get(nodeIndex);
+ }
+
+ public float getTotalRankChange()
+ {
+ return this.totalRankChange;
+ }
+
+ private void maybeCreateNode(int nodeId)
+ {
+ // create from node if it doesn't already exist
+ if (!nodeIndices.containsKey(nodeId))
+ {
+ int index = this.nodeData.size();
+
+ this.nodeData.add(0.0f); // rank
+ this.nodeData.add(0.0f); // total weight
+ this.nodeData.add(0.0f); // contribution
+
+ if (this.nodeBiasingEnabled)
+ {
+ this.nodeData.add(0.0f); // bias
+ }
+
+ this.nodeIndices.put(nodeId, index);
+
+ this.nodeCount++;
+ }
+ }
+
+ public float getNodeBias(int nodeId)
+ {
+ if (!this.nodeBiasingEnabled)
+ {
+ throw new IllegalArgumentException("Node biasing not enable");
+ }
+ int nodeIndex = this.nodeIndices.get(nodeId);
+ return this.nodeData.get(nodeIndex+3);
+ }
+
+ public void setNodeBias(int nodeId, float bias)
+ {
+ if (!this.nodeBiasingEnabled)
+ {
+ throw new IllegalArgumentException("Node biasing not enable");
+ }
+
+ int nodeIndex = this.nodeIndices.get(nodeId);
+ this.nodeData.set(nodeIndex+3, bias);
+ }
+
+ public void addNode(Integer sourceId, ArrayList<Map<String,Object>> sourceEdges) throws IOException
+ {
+ // with bias of 1.0, all nodes have an equal bias (that is, no bias)
+ addNode(sourceId, sourceEdges, 1.0f);
+ }
+
+ public void addNode(Integer sourceId, ArrayList<Map<String,Object>> sourceEdges, float bias) throws IOException
+ {
+ int source = sourceId.intValue();
+
+ maybeCreateNode(source);
+
+ if (this.nodeBiasingEnabled)
+ {
+ setNodeBias(source, bias);
+ }
+ else if (bias != 1.0f)
+ {
+ // with node biasing disabled, all nodes implicitly have a bias of 1.0, which means no bias, so if anything else was specified
+ // it won't take effect.
+ throw new IllegalArgumentException("Bias was specified but node biasing not enabled");
+ }
+
+ if (this.shouldCacheEdgesOnDisk && !usingEdgeDiskCache && (sourceEdges.size() + this.edgeCount) >= this.edgeCachingThreshold)
+ {
+ writeEdgesToDisk();
+ }
+
+ // store the source node id itself
+ appendEdgeData(source);
+
+ // store how many outgoing edges this node has
+ appendEdgeData(sourceEdges.size());
+
+ // store the outgoing edges
+ for (Map<String,Object> edge : sourceEdges)
+ {
+ int dest = ((Integer)edge.get("dest")).intValue();
+ float weight = ((Double)edge.get("weight")).floatValue();
+
+ maybeCreateNode(dest);
+
+ appendEdgeData(dest);
+
+ // location of weight in weights array
+ appendEdgeData(Math.max(1, (int)(weight * EDGE_WEIGHT_MULTIPLIER)));
+
+ this.edgeCount++;
+ }
+ }
+
+ private void appendEdgeData(int data) throws IOException
+ {
+ if (this.edgeDataOutputStream != null)
+ {
+ this.edgeDataOutputStream.writeInt(data);
+ }
+ else
+ {
+ this.edges.add(data);
+ }
+ }
+
+ public void init() throws IOException
+ {
+ init(getDummyIndicator());
+ }
+
+ public void init(ProgressIndicator progressIndicator) throws IOException
+ {
+ if (this.edgeDataOutputStream != null)
+ {
+ this.edgeDataOutputStream.close();
+ this.edgeDataOutputStream = null;
+ }
+
+ // initialize all nodes to an equal share of the total rank (1.0)
+ float nodeRank = 1.0f / this.nodeCount;
+ float totalBias = 0.0f;
+ for (int j=0; j<this.nodeData.size(); j+=this.nodeFieldCount)
+ {
+ nodeData.set(j, nodeRank);
+ progressIndicator.progress();
+ if (this.nodeBiasingEnabled)
+ {
+ totalBias += nodeData.getFloat(j+3);
+ }
+ }
+
+ // if node biasing enabled, need to normalize the bias by the total bias across all nodes so it represents
+ // the share of bias.
+ if (this.nodeBiasingEnabled)
+ {
+ for (int j=0; j<this.nodeData.size(); j+=this.nodeFieldCount)
+ {
+ float bias = nodeData.getFloat(j+3);
+ bias /= totalBias;
+ nodeData.set(j+3,bias);
+ }
+ }
+
+ Iterator<Integer> edgeData = getEdgeData();
+
+ while(edgeData.hasNext())
+ {
+ int sourceId = edgeData.next();
+ int nodeEdgeCount = edgeData.next();
+
+ while (nodeEdgeCount-- > 0)
+ {
+ // skip the destination node id
+ edgeData.next();
+
+ float weight = edgeData.next();
+
+ int nodeIndex = this.nodeIndices.get(sourceId);
+
+ float totalWeight = this.nodeData.getFloat(nodeIndex+1);
+ totalWeight += weight;
+ this.nodeData.set(nodeIndex+1, totalWeight);
+
+ progressIndicator.progress();
+ }
+ }
+
+ // if handling dangling nodes, get a list of them by finding those nodes with no outgoing
+ // edges (i.e. total outgoing edge weight is 0.0)
+ if (shouldHandleDanglingNodes)
+ {
+ for (Map.Entry<Integer,Integer> e : nodeIndices.entrySet())
+ {
+ int nodeId = e.getKey();
+ int nodeIndex = e.getValue();
+ float totalWeight = nodeData.getFloat(nodeIndex+1);
+ if (totalWeight == 0.0f)
+ {
+ danglingNodes.add(nodeId);
+ }
+ }
+ }
+ }
+
+ public float nextIteration(ProgressIndicator progressIndicator) throws IOException
+ {
+ distribute(progressIndicator);
+ commit(progressIndicator);
+
+ return getTotalRankChange();
+ }
+
+ public float nextIteration() throws IOException
+ {
+ ProgressIndicator dummyIndicator = getDummyIndicator();
+ distribute(dummyIndicator);
+ commit(dummyIndicator);
+
+ return getTotalRankChange();
+ }
+
+ private ProgressIndicator getDummyIndicator()
+ {
+ return new ProgressIndicator() {
+ @Override
+ public void progress()
+ {
+ }
+ };
+ }
+
+ public void distribute(ProgressIndicator progressIndicator) throws IOException
+ {
+ Iterator<Integer> edgeData = getEdgeData();
+
+ while(edgeData.hasNext())
+ {
+ int sourceId = edgeData.next();
+ int nodeEdgeCount = edgeData.next();
+
+ while (nodeEdgeCount-- > 0)
+ {
+ int toId = edgeData.next();
+ float weight = edgeData.next();
+
+ int fromNodeIndex = this.nodeIndices.get(sourceId);
+ int toNodeIndex = this.nodeIndices.get(toId);
+
+ float contributionChange = weight * this.nodeData.getFloat(fromNodeIndex) / this.nodeData.getFloat(fromNodeIndex+1);
+
+ float currentContribution = this.nodeData.getFloat(toNodeIndex+2);
+ this.nodeData.set(toNodeIndex+2, currentContribution + contributionChange);
+
+ progressIndicator.progress();
+ }
+ }
+
+ if (shouldHandleDanglingNodes)
+ {
+ // get the rank from each of the dangling nodes
+ float totalRank = 0.0f;
+ for (int nodeId : danglingNodes)
+ {
+ int nodeIndex = nodeIndices.get(nodeId);
+ float rank = nodeData.get(nodeIndex);
+ totalRank += rank;
+ }
+
+ // distribute the dangling node ranks to all the nodes in the graph
+ // note: the alpha factor is applied in the commit stage
+ float contributionIncrease = totalRank / this.nodeCount;
+ for (int i=2; i<nodeData.size(); i += this.nodeFieldCount)
+ {
+ float contribution = nodeData.getFloat(i);
+ contribution += contributionIncrease;
+ nodeData.set(i, contribution);
+ }
+ }
+ }
+
+ public void commit(ProgressIndicator progressIndicator)
+ {
+ this.totalRankChange = 0.0f;
+
+ float oneMinusAlpha = (1.0f - this.alpha);
+ float oneMinusAlphaOverNodeCount = oneMinusAlpha / nodeCount;
+
+ for (int nodeIndex=0; nodeIndex<this.nodeData.size(); nodeIndex += this.nodeFieldCount)
+ {
+ float oldRank = this.nodeData.get(nodeIndex+2);
+ float newRank;
+
+ if (this.nodeBiasingEnabled)
+ {
+ float bias = this.nodeData.get(nodeIndex+3);
+ newRank = bias * oneMinusAlpha + alpha * oldRank;
+ }
+ else
+ {
+ newRank = oneMinusAlphaOverNodeCount + alpha * oldRank;
+ }
+
+ this.nodeData.set(nodeIndex+2, 0.0f);
+
+ float lastRankDiff = newRank - this.nodeData.get(nodeIndex);
+
+ this.nodeData.set(nodeIndex, newRank);
+
+ this.totalRankChange += Math.abs(lastRankDiff);
+
+ progressIndicator.progress();
+ }
+ }
+
+ private void writeEdgesToDisk() throws IOException
+ {
+ this.edgesFile = File.createTempFile("fastgraph", null);
+
+ FileOutputStream outStream = new FileOutputStream(this.edgesFile);
+ BufferedOutputStream bufferedStream = new BufferedOutputStream(outStream);
+ this.edgeDataOutputStream = new DataOutputStream(bufferedStream);
+
+ for (int edgeData : edges)
+ {
+ this.edgeDataOutputStream.writeInt(edgeData);
+ }
+
+ this.edges.clear();
+ usingEdgeDiskCache = true;
+ }
+
+ private Iterator<Integer> getEdgeData() throws IOException
+ {
+ if (!usingEdgeDiskCache)
+ {
+ return this.edges.iterator();
+ }
+ else
+ {
+ FileInputStream fileInputStream = new FileInputStream(this.edgesFile);
+ BufferedInputStream inputStream = new BufferedInputStream(fileInputStream);
+ final DataInputStream dataInputStream = new DataInputStream(inputStream);
+
+ return new AbstractIterator<Integer>() {
+
+ @Override
+ protected Integer computeNext()
+ {
+ try
+ {
+ return dataInputStream.readInt();
+ }
+ catch (IOException e)
+ {
+ return endOfData();
+ }
+ }
+
+ };
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/linkanalysis/ProgressIndicator.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/linkanalysis/ProgressIndicator.java b/datafu-pig/src/main/java/datafu/pig/linkanalysis/ProgressIndicator.java
new file mode 100644
index 0000000..3dc54c5
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/linkanalysis/ProgressIndicator.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.linkanalysis;
+
+interface ProgressIndicator
+{
+ void progress();
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/linkanalysis/package-info.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/linkanalysis/package-info.java b/datafu-pig/src/main/java/datafu/pig/linkanalysis/package-info.java
new file mode 100644
index 0000000..2c6c078
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/linkanalysis/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * UDFs for performing link analysis, such as PageRank.
+ */
+package datafu.pig.linkanalysis;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/random/RandInt.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/random/RandInt.java b/datafu-pig/src/main/java/datafu/pig/random/RandInt.java
new file mode 100644
index 0000000..de89c4a
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/random/RandInt.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.random;
+
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.pig.builtin.Nondeterministic;
+import org.apache.pig.data.DataType;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+import datafu.pig.util.SimpleEvalFunc;
+
+/**
+ * Generates a uniformly distributed integer between two bounds.
+ */
+@Nondeterministic
+public class RandInt extends SimpleEvalFunc<Integer>
+{
+ private final Random rand = new Random();
+
+ /**
+ * @param min lower bound for random number
+ * @param max upper bound for random number
+ */
+ public Integer call(Integer min, Integer max) throws IOException
+ {
+ try
+ {
+ if (min > max)
+ {
+ throw new RuntimeException("The first argument must be less than the second");
+ }
+ return rand.nextInt(max - min + 1) + min;
+ }
+ catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ return new Schema(new Schema.FieldSchema("rand", DataType.INTEGER));
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/random/RandomUUID.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/random/RandomUUID.java b/datafu-pig/src/main/java/datafu/pig/random/RandomUUID.java
new file mode 100644
index 0000000..d63f4cf
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/random/RandomUUID.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package datafu.pig.random;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.pig.EvalFunc;
+import org.apache.pig.builtin.Nondeterministic;
+import org.apache.pig.data.*;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * Generates a random UUID using java.util.UUID
+ */
+@Nondeterministic
+public class RandomUUID extends EvalFunc<String>
+{
+ public String exec(Tuple input) throws IOException
+ {
+ return UUID.randomUUID().toString();
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ return new Schema(new Schema.FieldSchema("uuid", DataType.CHARARRAY));
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/random/package-info.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/random/package-info.java b/datafu-pig/src/main/java/datafu/pig/random/package-info.java
new file mode 100644
index 0000000..8c7750d
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/random/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * UDFs dealing with randomness.
+ */
+package datafu.pig.random;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/sampling/Reservoir.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/sampling/Reservoir.java b/datafu-pig/src/main/java/datafu/pig/sampling/Reservoir.java
new file mode 100644
index 0000000..09b4c95
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/sampling/Reservoir.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.sampling;
+
+import java.util.PriorityQueue;
+
+class Reservoir extends PriorityQueue<ScoredTuple>
+{
+ private static final long serialVersionUID = 1L;
+ private int numSamples;
+
+ public Reservoir(int numSamples)
+ {
+ super(numSamples);
+ this.numSamples = numSamples;
+ }
+
+ public boolean consider(ScoredTuple scoredTuple)
+ {
+ if (super.size() < numSamples) {
+ return super.add(scoredTuple);
+ } else {
+ ScoredTuple head = super.peek();
+ if (scoredTuple.score > head.score) {
+ super.poll();
+ return super.add(scoredTuple);
+ }
+ return false;
+ }
+ }
+}
[09/19] DATAFU-27 Migrate build system to Gradle
Posted by wv...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/bags/CountEach.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/bags/CountEach.java b/src/java/datafu/pig/bags/CountEach.java
deleted file mode 100644
index cfb0152..0000000
--- a/src/java/datafu/pig/bags/CountEach.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.bags;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.pig.AccumulatorEvalFunc;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * Generates a count of the number of times each distinct tuple appears in a bag.
- *
- * <p>
- * Example:
- * <pre>
- * {@code
- * DEFINE CountEach datafu.pig.bags.CountEach();
- * DEFINE CountEachFlatten datafu.pig.bags.CountEach('flatten');
- *
- * -- input:
- * -- ({(A),(A),(C),(B)})
- * input = LOAD 'input' AS (B: bag {T: tuple(alpha:CHARARRAY, numeric:INT)});
- *
- * -- output:
- * -- {((A),2),((C),1),((B),1)}
- * output = FOREACH input GENERATE CountEach(B);
- *
- * -- output_flatten:
- * -- ({(A,2),(C,1),(B,1)})
- * output_flatten = FOREACH input GENERATE CountEachFlatten(B);
- * }
- * </pre>
- * </p>
- */
-public class CountEach extends AccumulatorEvalFunc<DataBag>
-{
- private boolean flatten = false;
- private Map<Tuple, Integer> counts = new HashMap<Tuple, Integer>();
-
- public CountEach() {
-
- }
-
- public CountEach(String arg) {
- if (arg != null && arg.toLowerCase().equals("flatten")) {
- flatten = true;
- }
- }
-
- @Override
- public void accumulate(Tuple input) throws IOException
- {
- DataBag inputBag = (DataBag)input.get(0);
- if (inputBag == null) throw new IllegalArgumentException("Expected a bag, got null");
-
- for (Tuple tuple : inputBag) {
- if (!counts.containsKey(tuple)) {
- counts.put(tuple, 0);
- }
- counts.put(tuple, counts.get(tuple)+1);
- }
- }
-
- @Override
- public DataBag getValue()
- {
- DataBag output = BagFactory.getInstance().newDefaultBag();
- for (Tuple tuple : counts.keySet()) {
- Tuple outputTuple = null;
- Tuple innerTuple = TupleFactory.getInstance().newTuple(tuple.getAll());
- if (flatten) {
- innerTuple.append(counts.get(tuple));
- outputTuple = innerTuple;
- } else {
- outputTuple = TupleFactory.getInstance().newTuple();
- outputTuple.append(innerTuple);
- outputTuple.append(counts.get(tuple));
- }
- output.add(outputTuple);
- }
-
- return output;
- }
-
- @Override
- public void cleanup()
- {
- counts.clear();
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try {
- if (input.size() != 1)
- {
- throw new RuntimeException("Expected input to have one field");
- }
-
- Schema.FieldSchema bagFieldSchema = input.getField(0);
-
- if (bagFieldSchema.type != DataType.BAG)
- {
- throw new RuntimeException("Expected a BAG as input");
- }
-
- Schema inputBagSchema = bagFieldSchema.schema;
-
- if (inputBagSchema.getField(0).type != DataType.TUPLE)
- {
- throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
- DataType.findTypeName(inputBagSchema.getField(0).type)));
- }
-
- Schema inputTupleSchema = inputBagSchema.getField(0).schema;
- if (inputTupleSchema == null) inputTupleSchema = new Schema();
-
- Schema outputTupleSchema = null;
-
- if (this.flatten) {
- outputTupleSchema = inputTupleSchema.clone();
- outputTupleSchema.add(new Schema.FieldSchema("count", DataType.INTEGER));
- } else {
- outputTupleSchema = new Schema();
- outputTupleSchema.add(new Schema.FieldSchema("tuple_schema", inputTupleSchema.clone(), DataType.TUPLE));
- outputTupleSchema.add(new Schema.FieldSchema("count", DataType.INTEGER));
- }
-
- return new Schema(new Schema.FieldSchema(
- getSchemaName(this.getClass().getName().toLowerCase(), input),
- outputTupleSchema,
- DataType.BAG));
- }
- catch (CloneNotSupportedException e) {
- throw new RuntimeException(e);
- }
- catch (FrontendException e) {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/bags/DistinctBy.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/bags/DistinctBy.java b/src/java/datafu/pig/bags/DistinctBy.java
deleted file mode 100644
index a79e4de..0000000
--- a/src/java/datafu/pig/bags/DistinctBy.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.bags;
-
-import java.io.IOException;
-import java.util.HashSet;
-
-import org.apache.pig.AccumulatorEvalFunc;
-import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * Get distinct elements in a bag by a given set of field positions.
- * The input and output schemas will be identical.
- *
- * The first tuple containing each distinct combination of these fields will be taken.
- *
- * This operation is order preserving. If both A and B appear in the output,
- * and A appears before B in the input, then A will appear before B in the output.
- *
- * Example:
- * <pre>
- * {@code
- * define DistinctBy datafu.pig.bags.DistinctBy('0');
- *
- * -- input:
- * -- ({(a, 1),(a,1),(b, 2),(b,22),(c, 3),(d, 4)})
- * input = LOAD 'input' AS (B: bag {T: tuple(alpha:CHARARRAY, numeric:INT)});
- *
- * output = FOREACH input GENERATE DistinctBy(B);
- *
- * -- output:
- * -- ({(a,1),(b,2),(c,3),(d,4)})
- * }
- * </pre>
- *
- * @param map Any number of strings specifying field positions
- */
-public class DistinctBy extends AccumulatorEvalFunc<DataBag>
-{
- private HashSet<Integer> fields = new HashSet<Integer>();
- private HashSet<Tuple> seen = new HashSet<Tuple>();
- private DataBag outputBag;
-
- public DistinctBy(String... fields)
- {
- for(String field : fields) {
- this.fields.add(Integer.parseInt(field));
- }
- cleanup();
- }
-
- @Override
- public void accumulate(Tuple input) throws IOException
- {
- if (input.size() != 1) {
- throw new RuntimeException("Expected input to have only a single field");
- }
- if (input.getType(0) != DataType.BAG) {
- throw new RuntimeException("Expected a BAG as input");
- }
-
- DataBag inputBag = (DataBag)input.get(0);
- for (Tuple t : inputBag) {
- Tuple distinctFieldTuple = getDistinctFieldTuple(t, this.fields);
- if (!seen.contains(distinctFieldTuple)) {
- outputBag.add(t);
- seen.add(distinctFieldTuple);
- }
- }
- }
-
- @Override
- public void cleanup()
- {
- seen.clear();
- outputBag = BagFactory.getInstance().newDefaultBag();
- }
-
- @Override
- public DataBag getValue()
- {
- return outputBag;
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try {
- if (input.size() != 1)
- {
- throw new RuntimeException("Expected input to have only a single field");
- }
-
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.BAG)
- {
- throw new RuntimeException("Expected a BAG as input");
- }
-
- Schema inputBagSchema = inputFieldSchema.schema;
-
- if (inputBagSchema.getField(0).type != DataType.TUPLE)
- {
- throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
- DataType.findTypeName(inputBagSchema.getField(0).type)));
- }
-
- Schema inputTupleSchema = inputBagSchema.getField(0).schema;
-
- Schema outputTupleSchema = inputTupleSchema.clone();
-
- return new Schema(new Schema.FieldSchema(
- getSchemaName(this.getClass().getName().toLowerCase(), input),
- outputTupleSchema,
- DataType.BAG));
- }
- catch (CloneNotSupportedException e) {
- throw new RuntimeException(e);
- }
- catch (FrontendException e) {
- throw new RuntimeException(e);
- }
- }
-
- private Tuple getDistinctFieldTuple(Tuple t, HashSet<Integer> distinctFieldPositions) throws ExecException {
- Tuple fieldTuple = TupleFactory.getInstance().newTuple(distinctFieldPositions.size());
- int idx = 0;
- for(int i=0; i<t.size(); i++) {
- if (distinctFieldPositions.contains(i)) {
- fieldTuple.set(idx, t.get(i));
- idx++;
- }
- }
- return fieldTuple;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/bags/EmptyBagToNull.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/bags/EmptyBagToNull.java b/src/java/datafu/pig/bags/EmptyBagToNull.java
deleted file mode 100644
index 5524ad5..0000000
--- a/src/java/datafu/pig/bags/EmptyBagToNull.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.bags;
-
-import java.io.IOException;
-
-import org.apache.pig.EvalFunc;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * Returns null if the input is an empty bag; otherwise,
- * returns the input bag unchanged.
- */
-public class EmptyBagToNull extends EvalFunc<DataBag>
-{
- @Override
- public DataBag exec(Tuple tuple) throws IOException
- {
- if (tuple.size() == 0 || tuple.get(0) == null)
- return null;
- Object o = tuple.get(0);
- if (o instanceof DataBag)
- {
- DataBag bag = (DataBag)o;
- if (bag.size() == 0)
- {
- return null;
- }
- else
- {
- return bag;
- }
- }
- else
- throw new IllegalArgumentException("expected a null or a bag");
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- return input;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/bags/EmptyBagToNullFields.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/bags/EmptyBagToNullFields.java b/src/java/datafu/pig/bags/EmptyBagToNullFields.java
deleted file mode 100644
index 6933c28..0000000
--- a/src/java/datafu/pig/bags/EmptyBagToNullFields.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.bags;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-import datafu.pig.util.ContextualEvalFunc;
-
-/**
- * For an empty bag, inserts a tuple having null values for all fields;
- * otherwise, the input bag is returned unchanged.
- *
- * <p>
- * This can be useful when performing FLATTEN on a bag from a COGROUP,
- * as FLATTEN on an empty bag produces no data.
- * </p>
- */
-public class EmptyBagToNullFields extends ContextualEvalFunc<DataBag>
-{
- @Override
- public DataBag exec(Tuple tuple) throws IOException
- {
- if (tuple.size() == 0 || tuple.get(0) == null)
- return null;
- Object o = tuple.get(0);
- if (o instanceof DataBag)
- {
- DataBag bag = (DataBag)o;
- if (bag.size() == 0)
- {
- // create a tuple with null values for all fields
- int tupleSize = (Integer)getInstanceProperties().get("tuplesize");
- return BagFactory.getInstance().newDefaultBag(Arrays.asList(TupleFactory.getInstance().newTuple(tupleSize)));
- }
- else
- {
- return bag;
- }
- }
- else
- throw new IllegalArgumentException("expected a null or a bag");
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try
- {
- if (input.size() != 1)
- {
- throw new RuntimeException("Expected only a single field as input");
- }
-
- if (input.getField(0).type != DataType.BAG)
- {
- throw new RuntimeException("Expected a BAG as input, but found " + DataType.findTypeName(input.getField(0).type));
- }
-
- // get the size of the tuple within the bag
- int innerTupleSize = input.getField(0).schema.getField(0).schema.getFields().size();
- getInstanceProperties().put("tuplesize", innerTupleSize);
- }
- catch (FrontendException e)
- {
- throw new RuntimeException(e);
- }
- return input;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/bags/Enumerate.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/bags/Enumerate.java b/src/java/datafu/pig/bags/Enumerate.java
deleted file mode 100644
index 8a0d072..0000000
--- a/src/java/datafu/pig/bags/Enumerate.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.bags;
-
-import java.io.IOException;
-
-import org.apache.pig.AccumulatorEvalFunc;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * Enumerate a bag, appending to each tuple its index within the bag.
- *
- * <p>
- * For example:
- * <pre>
- * {(A),(B),(C),(D)} => {(A,0),(B,1),(C,2),(D,3)}
- * </pre>
- * The first constructor parameter (optional) dictates the starting index of the counting.
- * This UDF implements the accumulator interface, reducing DataBag materialization costs.
- * </p>
- *
- * <p>
- * Example:
- * <pre>
- * {@code
- * define Enumerate datafu.pig.bags.Enumerate('1');
- *
- * -- input:
- * -- ({(100),(200),(300),(400)})
- * input = LOAD 'input' as (B: bag{T: tuple(v2:INT)});
- *
- * -- output:
- * -- ({(100,1),(200,2),(300,3),(400,4)})
- * output = FOREACH input GENERATE Enumerate(B);
- * }
- * </pre>
- */
-public class Enumerate extends AccumulatorEvalFunc<DataBag>
-{
- private final int start;
-
- private DataBag outputBag;
- private long i;
- private long count;
-
- public Enumerate()
- {
- this("0");
- }
-
- public Enumerate(String start)
- {
- this.start = Integer.parseInt(start);
- cleanup();
- }
-
- @Override
- public void accumulate(Tuple arg0) throws IOException
- {
- DataBag inputBag = (DataBag)arg0.get(0);
- for (Tuple t : inputBag) {
- Tuple t1 = TupleFactory.getInstance().newTuple(t.getAll());
- t1.append(i);
- outputBag.add(t1);
-
- if (count % 1000000 == 0) {
- outputBag.spill();
- count = 0;
- }
- i++;
- count++;
- }
- }
-
- @Override
- public void cleanup()
- {
- this.outputBag = BagFactory.getInstance().newDefaultBag();
- this.i = this.start;
- this.count = 0;
- }
-
- @Override
- public DataBag getValue()
- {
- return outputBag;
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try {
- if (input.size() != 1)
- {
- throw new RuntimeException("Expected input to have only a single field");
- }
-
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.BAG)
- {
- throw new RuntimeException("Expected a BAG as input");
- }
-
- Schema inputBagSchema = inputFieldSchema.schema;
-
- if (inputBagSchema.getField(0).type != DataType.TUPLE)
- {
- throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
- DataType.findTypeName(inputBagSchema.getField(0).type)));
- }
-
- Schema inputTupleSchema = inputBagSchema.getField(0).schema;
-
- Schema outputTupleSchema = inputTupleSchema.clone();
- outputTupleSchema.add(new Schema.FieldSchema("i", DataType.LONG));
-
- return new Schema(new Schema.FieldSchema(
- getSchemaName(this.getClass().getName().toLowerCase(), input),
- outputTupleSchema,
- DataType.BAG));
- }
- catch (CloneNotSupportedException e) {
- throw new RuntimeException(e);
- }
- catch (FrontendException e) {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/bags/FirstTupleFromBag.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/bags/FirstTupleFromBag.java b/src/java/datafu/pig/bags/FirstTupleFromBag.java
deleted file mode 100644
index 1f24984..0000000
--- a/src/java/datafu/pig/bags/FirstTupleFromBag.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.bags;
-
-import java.io.IOException;
-
-import datafu.pig.util.SimpleEvalFunc;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * Returns the first tuple from a bag. Requires a second parameter that will be returned if the bag is empty.
- *
- * Example:
- * <pre>
- * {@code
- * define FirstTupleFromBag datafu.pig.bags.FirstTupleFromBag();
- *
- * -- input:
- * -- ({(a,1)})
- * input = LOAD 'input' AS (B: bag {T: tuple(alpha:CHARARRAY, numeric:INT)});
- *
- * output = FOREACH input GENERATE FirstTupleFromBag(B, null);
- *
- * -- output:
- * -- (a,1)
- * }
- * </pre>
- */
-
-public class FirstTupleFromBag extends SimpleEvalFunc<Tuple>
-{
- public Tuple call(DataBag bag, Tuple defaultValue) throws IOException
- {
- for (Tuple t : bag) {
- return t;
- }
- return defaultValue;
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try {
- return new Schema(input.getField(0).schema);
- }
- catch (Exception e) {
- return null;
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/bags/NullToEmptyBag.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/bags/NullToEmptyBag.java b/src/java/datafu/pig/bags/NullToEmptyBag.java
deleted file mode 100644
index 09fffb3..0000000
--- a/src/java/datafu/pig/bags/NullToEmptyBag.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.bags;
-
-import java.io.IOException;
-
-import org.apache.pig.EvalFunc;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * Returns an empty bag if the input is null; otherwise,
- * returns the input bag unchanged.
- */
-public class NullToEmptyBag extends EvalFunc<DataBag>
-{
- @Override
- public DataBag exec(Tuple tuple) throws IOException
- {
- if (tuple.size() == 0 || tuple.get(0) == null)
- return BagFactory.getInstance().newDefaultBag();
- Object o = tuple.get(0);
- if (o instanceof DataBag)
- return (DataBag)o;
- else
- throw new IllegalArgumentException("expected a null or a bag");
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- return input;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/bags/PrependToBag.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/bags/PrependToBag.java b/src/java/datafu/pig/bags/PrependToBag.java
deleted file mode 100644
index 9292871..0000000
--- a/src/java/datafu/pig/bags/PrependToBag.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.bags;
-
-import java.io.IOException;
-
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-import datafu.pig.util.SimpleEvalFunc;
-
-/**
- * Prepends a tuple to a bag.
- *
- * <p>N.B. this copies the entire input bag, so don't use it for large bags.</p>
- *
- * <p>
- * Example:
- * <pre>
- * {@code
- * define PrependToBag datafu.pig.bags.PrependToBag();
- *
- * -- input:
- * -- ({(1),(2),(3)},(4))
- * -- ({(10),(20),(30),(40),(50)},(60))
- * input = LOAD 'input' AS (B: bag{T: tuple(v:INT)}, T: tuple(v:INT));
-
- * -- output:
- * -- ({(4),(1),(2),(3)})
- * -- ({(60),(10),(20),(30),(40),(50)})
- * output = FOREACH input GENERATE PrependToBag(B,T) as B;
- * }
- * </pre>
- * </p>
- */
-public class PrependToBag extends SimpleEvalFunc<DataBag>
-{
- public DataBag call(DataBag inputBag, Tuple t) throws IOException
- {
- DataBag outputBag = BagFactory.getInstance().newDefaultBag();
- outputBag.add(t);
- for (Tuple x : inputBag)
- outputBag.add(x);
- return outputBag;
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try {
- return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
- input.getField(0).schema, DataType.BAG));
- }
- catch (FrontendException e) {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/bags/ReverseEnumerate.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/bags/ReverseEnumerate.java b/src/java/datafu/pig/bags/ReverseEnumerate.java
deleted file mode 100644
index c86ffcf..0000000
--- a/src/java/datafu/pig/bags/ReverseEnumerate.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.bags;
-
-import java.io.IOException;
-
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-import datafu.pig.util.SimpleEvalFunc;
-
-/**
- * Enumerate a bag, appending to each tuple its index within the bag, with indices being produced in
- * descending order.
- *
- * <p>
- * For example:
- * <pre>
- * {(A),(B),(C),(D)} => {(A,3),(B,2),(C,1),(D,0)}
- * </pre>
- * The first constructor parameter (optional) dictates the starting index of the counting. As the
- * UDF requires the size of the bag for reverse counting, this UDF does <b>not</b> implement the
- * accumulator interface and suffers from the slight performance penalty of DataBag materialization.
- * </p>
- *
- * <p>
- * Example:
- * <pre>
- * {@code
- * define ReverseEnumerate datafu.pig.bags.ReverseEnumerate('1');
- *
- * -- input:
- * -- ({(100),(200),(300),(400)})
- * input = LOAD 'input' as (B: bag{T: tuple(v2:INT)});
- *
- * -- output:
- * -- ({(100,4),(200,3),(300,2),(400,1)})
- * output = FOREACH input GENERATE ReverseEnumerate(B);
- * }
- * </pre>
- * </p>
- */
-public class ReverseEnumerate extends SimpleEvalFunc<DataBag>
-{
- private final int start;
-
- public ReverseEnumerate()
- {
- this.start = 0;
- }
-
- public ReverseEnumerate(String start)
- {
- this.start = Integer.parseInt(start);
- }
-
- public DataBag call(DataBag inputBag) throws IOException
- {
- DataBag outputBag = BagFactory.getInstance().newDefaultBag();
- long i = start, count = 0;
- i = inputBag.size() - 1 + start;
-
- for (Tuple t : inputBag) {
- Tuple t1 = TupleFactory.getInstance().newTuple(t.getAll());
- t1.append(i);
- outputBag.add(t1);
-
- if (count % 1000000 == 0) {
- outputBag.spill();
- count = 0;
- }
- i--;
- count++;
- }
-
- return outputBag;
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try {
- if (input.size() != 1)
- {
- throw new RuntimeException("Expected input to have only a single field");
- }
-
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.BAG)
- {
- throw new RuntimeException("Expected a BAG as input");
- }
-
- Schema inputBagSchema = inputFieldSchema.schema;
-
- if (inputBagSchema.getField(0).type != DataType.TUPLE)
- {
- throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
- DataType.findTypeName(inputBagSchema.getField(0).type)));
- }
-
- Schema inputTupleSchema = inputBagSchema.getField(0).schema;
-
- Schema outputTupleSchema = inputTupleSchema.clone();
- outputTupleSchema.add(new Schema.FieldSchema("i", DataType.LONG));
-
- return new Schema(new Schema.FieldSchema(
- getSchemaName(this.getClass().getName().toLowerCase(), input),
- outputTupleSchema,
- DataType.BAG));
- }
- catch (CloneNotSupportedException e) {
- throw new RuntimeException(e);
- }
- catch (FrontendException e) {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/bags/UnorderedPairs.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/bags/UnorderedPairs.java b/src/java/datafu/pig/bags/UnorderedPairs.java
deleted file mode 100644
index a1d149e..0000000
--- a/src/java/datafu/pig/bags/UnorderedPairs.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.bags;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.pig.EvalFunc;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.tools.pigstats.PigStatusReporter;
-
-/**
- * Generates pairs of all items in a bag.
- *
- * <p>
- * Example:
- * <pre>
- * {@code
- * define UnorderedPairs datafu.pig.bags.UnorderedPairs();
- *
- * -- input:
- * -- ({(1),(2),(3),(4)})
- * input = LOAD 'input' AS (B: bag {T: tuple(v:INT)});
- *
- * -- output:
- * -- ({((1),(2)),((1),(3)),((1),(4)),((2),(3)),((2),(4)),((3),(4))})
- * output = FOREACH input GENERATE UnorderedPairs(B);
- * }
- * </pre>
- * </p>
- */
-public class UnorderedPairs extends EvalFunc<DataBag>
-{
- private static final BagFactory bagFactory = BagFactory.getInstance();
- private static final TupleFactory tupleFactory = TupleFactory.getInstance();
-
- @Override
- public DataBag exec(Tuple input) throws IOException
- {
- PigStatusReporter reporter = PigStatusReporter.getInstance();
-
- try {
- DataBag inputBag = (DataBag) input.get(0);
- DataBag outputBag = bagFactory.newDefaultBag();
- long i=0, j, cnt=0;
-
- if (inputBag != null)
- {
- for (Tuple elem1 : inputBag) {
- j = 0;
- for (Tuple elem2 : inputBag) {
- if (j > i) {
- outputBag.add(tupleFactory.newTuple(Arrays.asList(elem1, elem2)));
- cnt++;
- }
- j++;
-
- if (reporter != null)
- reporter.progress();
-
- if (cnt % 1000000 == 0) {
- outputBag.spill();
- cnt = 0;
- }
- }
- i++;
- }
- }
-
- return outputBag;
- }
- catch (Exception e) {
- throw new RuntimeException("Caught exception processing input of " + this.getClass().getName(), e);
- }
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try {
- if (input.size() != 1)
- {
- throw new RuntimeException("Expected input to have only a single field");
- }
-
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.BAG)
- {
- throw new RuntimeException("Expected a BAG as input");
- }
-
- Schema inputBagSchema = inputFieldSchema.schema;
-
- if (inputBagSchema.getField(0).type != DataType.TUPLE)
- {
- throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
- DataType.findTypeName(inputBagSchema.getField(0).type)));
- }
-
- Schema ouputTupleSchema = new Schema();
- ouputTupleSchema.add(new Schema.FieldSchema("elem1", inputBagSchema.getField(0).schema.clone(), DataType.TUPLE));
- ouputTupleSchema.add(new Schema.FieldSchema("elem2", inputBagSchema.getField(0).schema.clone(), DataType.TUPLE));
- return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
- ouputTupleSchema,
- DataType.BAG));
- }
- catch (Exception e) {
- return null;
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/bags/package-info.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/bags/package-info.java b/src/java/datafu/pig/bags/package-info.java
deleted file mode 100644
index 214b837..0000000
--- a/src/java/datafu/pig/bags/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * A collection of general purpose UDFs for operating on bags.
- */
-package datafu.pig.bags;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/geo/HaversineDistInMiles.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/geo/HaversineDistInMiles.java b/src/java/datafu/pig/geo/HaversineDistInMiles.java
deleted file mode 100644
index d1e3988..0000000
--- a/src/java/datafu/pig/geo/HaversineDistInMiles.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.geo;
-
-import org.apache.pig.data.DataType;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-import datafu.pig.util.SimpleEvalFunc;
-
-/**
- * Computes the distance (in miles) between two latitude-longitude pairs
- * using the {@link <a href="http://en.wikipedia.org/wiki/Haversine_formula" target="_blank">Haversine formula</a>}.
- *
- * <p>
- * Example:
- * <pre>
- * {@code
- * -- input is a TSV of two latitude and longitude pairs
- * input = LOAD 'input' AS (lat1 : double, long1 : double, lat2 : double, long2 : double);
- * output = FOREACH input GENERATE datafu.pig.geo.HaversineDistInMiles(lat1, long1, lat2, long2) as distance;
- * }</pre></p>
- */
-public class HaversineDistInMiles extends SimpleEvalFunc<Double>
-{
- public static final double EARTH_RADIUS = 3958.75;
-
- public Double call(Double lat1, Double lng1, Double lat2, Double lng2)
- {
- if (lat1 == null || lng1 == null || lat2 == null || lng2 == null)
- return null;
-
- double d_lat = Math.toRadians(lat2-lat1);
- double d_long = Math.toRadians(lng2-lng1);
- double a = Math.sin(d_lat/2) * Math.sin(d_lat/2) +
- Math.cos(Math.toRadians(lat1)) * Math.cos(Math.toRadians(lat2)) *
- Math.sin(d_long/2) * Math.sin(d_long/2);
- double c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1-a));
- return EARTH_RADIUS * c;
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- return new Schema(new Schema.FieldSchema("dist", DataType.DOUBLE));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/geo/package-info.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/geo/package-info.java b/src/java/datafu/pig/geo/package-info.java
deleted file mode 100644
index 12b27b0..0000000
--- a/src/java/datafu/pig/geo/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * UDFs for geographic computations.
- */
-package datafu.pig.geo;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/hash/MD5.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/hash/MD5.java b/src/java/datafu/pig/hash/MD5.java
deleted file mode 100644
index b7b51da..0000000
--- a/src/java/datafu/pig/hash/MD5.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.hash;
-
-import java.math.BigInteger;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-
-import org.apache.commons.codec.binary.Base64;
-
-import datafu.pig.util.SimpleEvalFunc;
-
-/**
- * Computes the MD5 value of a string and outputs it in hex (by default).
- * A method can be provided to the constructor, which may be either 'hex' or 'base64'.
- */
-public class MD5 extends SimpleEvalFunc<String>
-{
- private final MessageDigest md5er;
- private final boolean isBase64;
-
- public MD5()
- {
- this("hex");
- }
-
- public MD5(String method)
- {
- if ("hex".equals(method))
- {
- isBase64 = false;
- }
- else if ("base64".equals(method))
- {
- isBase64 = true;
- }
- else
- {
- throw new IllegalArgumentException("Expected either hex or base64");
- }
-
- try {
- md5er = MessageDigest.getInstance("md5");
- }
- catch (NoSuchAlgorithmException e) {
- throw new RuntimeException(e);
- }
- }
-
- public String call(String val)
- {
- if (isBase64)
- {
- return new String(Base64.encodeBase64(md5er.digest(val.getBytes())));
- }
- else
- {
- return new BigInteger(1, md5er.digest(val.getBytes())).toString(16);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/hash/SHA.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/hash/SHA.java b/src/java/datafu/pig/hash/SHA.java
deleted file mode 100644
index ff859e5..0000000
--- a/src/java/datafu/pig/hash/SHA.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.hash;
-
-import java.math.BigInteger;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-
-import datafu.pig.util.SimpleEvalFunc;
-
-public class SHA extends SimpleEvalFunc<String> {
- private final MessageDigest sha;
-
- public SHA(){
- this("256");
- }
-
- public SHA(String algorithm){
- try {
- sha = MessageDigest.getInstance("SHA-"+algorithm);
- } catch (NoSuchAlgorithmException e) {
- throw new RuntimeException(e);
- }
- }
-
- public String call(String value){
- return new BigInteger(1, sha.digest(value.getBytes())).toString(16);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/hash/package-info.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/hash/package-info.java b/src/java/datafu/pig/hash/package-info.java
deleted file mode 100644
index 320a029..0000000
--- a/src/java/datafu/pig/hash/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * UDFs for computing hashes from data.
- */
-package datafu.pig.hash;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/linkanalysis/PageRank.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/linkanalysis/PageRank.java b/src/java/datafu/pig/linkanalysis/PageRank.java
deleted file mode 100644
index 80ff567..0000000
--- a/src/java/datafu/pig/linkanalysis/PageRank.java
+++ /dev/null
@@ -1,482 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.linkanalysis;
-
-import it.unimi.dsi.fastutil.ints.Int2IntMap;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.pig.Accumulator;
-import org.apache.pig.AccumulatorEvalFunc;
-import org.apache.pig.EvalFunc;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * A UDF which implements {@link <a href="http://en.wikipedia.org/wiki/PageRank" target="_blank">PageRank</a>}.
- *
- * <p>
- * This is not a distributed implementation. Each graph is stored in memory while running the algorithm, with edges optionally
- * spilled to disk to conserve memory. This can be used to distribute the execution of PageRank on multiple
- * reasonably sized graphs. It does not distribute execuion of PageRank for each individual graph. Each graph is identified
- * by an integer valued topic ID.
- * </p>
- *
- * <p>
- * If the graph is too large to fit in memory than an alternative method must be used, such as an iterative approach which runs
- * many MapReduce jobs in a sequence to complete the PageRank iterations.
- * </p>
- *
- * <p>
- * Each graph is represented through a bag of (source,edges) tuples. The 'source' is an integer ID representing the source node.
- * The 'edges' are the outgoing edges from the source node, represented as a bag of (dest,weight) tuples. The 'dest' is an
- * integer ID representing the destination node. The weight is a double representing how much the edge should be weighted.
- * For a standard PageRank implementation just use weight of 1.0.
- * </p>
- *
- * <p>
- * The output of the UDF is a bag of (source,rank) pairs, where 'rank' is the PageRank value for that source in the graph.
- * </p>
- *
- * <p>
- * There are several configurable options for this UDF, among them:
- * <p>
- *
- * <ul>
- * <li>
- * <b>alpha</b>: Controls the PageRank alpha value. The default is 0.85. A higher value reduces the "random jump"
- * factor and causes the rank to be influenced more by edges.
- * </li>
- * <li>
- * <b>max_iters</b>: The maximum number of iterations to run. The default is 150.
- * </li>
- * <li>
- * <b>dangling_nodes</b>: How to handling "dangling nodes", i.e. nodes with no outgoing edges. When "true" this is equivalent
- * to forcing a dangling node to have an outgoing edge to every other node in the graph. The default is "false".
- * </li>
- * <li>
- * <b>tolerance</b>: A threshold which causes iterations to cease. It is measured from the total change in ranks from each of
- * the nodes in the graph. As the ranks settle on their final values the total change decreases. This can be used
- * to stop iterations early. The default is 1e-16.
- * </li>
- * <li>
- * <b>max_nodes_and_edges</b>: This is a control to prevent running out of memory. As a graph is loaded, if the sum of edges
- * and nodes exceeds this value then it will stop. It will not fail but PageRank will not be run on this graph. Instead a null
- * value will be returned as a result. The default is 100M.
- * </li>
- * <li>
- * <b>spill_to_edge_disk_storage</b>: Used to conserve memory. When "true" it causes the edge data to be written to disk in a temp file instead
- * of being held in memory when the number of edges exceeds a threshold. The nodes are still held in memory however.
- * Each iteration of PageRank will stream through the edges stored on disk. The default is "false".
- * </li>
- * <li>
- * <b>max_edges_in_memory</b>: When spilling edges to disk is enabled, this is the threshold which triggers that behavior. The default is 30M.
- * </li>
- * </ul>
- *
- * <p>
- * Parameters are configured by passing them in as a sequence of pairs into the UDF constructor. For example, below the alpha value is set to
- * 0.87 and dangling nodes are enabled. All arguments must be strings.
- * </p>
- *
- * <p>
- * <pre>
- * {@code
- * define PageRank datafu.pig.linkanalysis.PageRank('alpha','0.87','dangling_nodes','true');
- * }
- * </pre>
- * </p>
- *
- * <p>
- * Full example:
- * <pre>
- * {@code
- *
- * topic_edges = LOAD 'input_edges' as (topic:INT,source:INT,dest:INT,weight:DOUBLE);
- *
- * topic_edges_grouped = GROUP topic_edges by (topic, source) ;
- * topic_edges_grouped = FOREACH topic_edges_grouped GENERATE
- * group.topic as topic,
- * group.source as source,
- * topic_edges.(dest,weight) as edges;
- *
- * topic_edges_grouped_by_topic = GROUP topic_edges_grouped BY topic;
- *
- * topic_ranks = FOREACH topic_edges_grouped_by_topic GENERATE
- * group as topic,
- * FLATTEN(PageRank(topic_edges_grouped.(source,edges))) as (source,rank);
- *
- * topic_ranks = FOREACH topic_ranks GENERATE
- * topic, source, rank;
- *
- * }
- * </pre>
- * </p>
- */
-public class PageRank extends AccumulatorEvalFunc<DataBag>
-{
- private final datafu.pig.linkanalysis.PageRankImpl graph = new datafu.pig.linkanalysis.PageRankImpl();
-
- private int maxNodesAndEdges = 100000000;
- private int maxEdgesInMemory = 30000000;
- private double tolerance = 1e-16;
- private int maxIters = 150;
- private boolean useEdgeDiskStorage = false;
- private boolean enableDanglingNodeHandling = false;
- private boolean enableNodeBiasing = false;
- private boolean aborted = false;
- private float alpha = 0.85f;
-
- TupleFactory tupleFactory = TupleFactory.getInstance();
- BagFactory bagFactory = BagFactory.getInstance();
-
- public PageRank()
- {
- initialize();
- }
-
- public PageRank(String... parameters)
- {
- if (parameters.length % 2 != 0)
- {
- throw new RuntimeException("Invalid parameters list");
- }
-
- for (int i=0; i<parameters.length; i+=2)
- {
- String parameterName = parameters[i];
- String value = parameters[i+1];
- if (parameterName.equals("max_nodes_and_edges"))
- {
- maxNodesAndEdges = Integer.parseInt(value);
- }
- else if (parameterName.equals("max_edges_in_memory"))
- {
- maxEdgesInMemory = Integer.parseInt(value);
- }
- else if (parameterName.equals("tolerance"))
- {
- tolerance = Double.parseDouble(value);
- }
- else if (parameterName.equals("max_iters"))
- {
- maxIters = Integer.parseInt(value);
- }
- else if (parameterName.equals("spill_to_edge_disk_storage"))
- {
- useEdgeDiskStorage = Boolean.parseBoolean(value);
- }
- else if (parameterName.equals("dangling_nodes"))
- {
- enableDanglingNodeHandling = Boolean.parseBoolean(value);
- }
- else if (parameterName.equals("node_biasing"))
- {
- enableNodeBiasing = Boolean.parseBoolean(value);
- }
- else if (parameterName.equals("alpha"))
- {
- alpha = Float.parseFloat(value);
- }
- }
-
- initialize();
- }
-
- private void initialize()
- {
- if (useEdgeDiskStorage)
- {
- this.graph.enableEdgeDiskCaching();
- }
- else
- {
- this.graph.disableEdgeDiskCaching();
- }
-
- if (enableDanglingNodeHandling)
- {
- this.graph.enableDanglingNodeHandling();
- }
- else
- {
- this.graph.disableDanglingNodeHandling();
- }
-
- if (enableNodeBiasing)
- {
- this.graph.enableNodeBiasing();
- }
- else
- {
- this.graph.disableNodeBiasing();
- }
-
- this.graph.setEdgeCachingThreshold(maxEdgesInMemory);
- this.graph.setAlpha(alpha);
- }
-
- @Override
- public void accumulate(Tuple t) throws IOException
- {
- if (aborted)
- {
- return;
- }
-
- DataBag bag = (DataBag) t.get(0);
- if (bag == null || bag.size() == 0)
- return;
-
- for (Tuple sourceTuple : bag)
- {
- Integer sourceId = (Integer)sourceTuple.get(0);
- DataBag edges = (DataBag)sourceTuple.get(1);
- Double nodeBias = null;
- if (enableNodeBiasing)
- {
- nodeBias = (Double)sourceTuple.get(2);
- }
-
- ArrayList<Map<String,Object>> edgesMapList = new ArrayList<Map<String, Object>>();
-
- for (Tuple edgeTuple : edges)
- {
- Integer destId = (Integer)edgeTuple.get(0);
- Double weight = (Double)edgeTuple.get(1);
- HashMap<String,Object> edgeMap = new HashMap<String, Object>();
- edgeMap.put("dest",destId);
- edgeMap.put("weight",weight);
- edgesMapList.add(edgeMap);
- }
-
- if (enableNodeBiasing)
- {
- graph.addNode(sourceId, edgesMapList, nodeBias.floatValue());
- }
- else
- {
- graph.addNode(sourceId, edgesMapList);
- }
-
- if (graph.nodeCount() + graph.edgeCount() > maxNodesAndEdges)
- {
- System.out.println(String.format("There are too many nodes and edges (%d + %d > %d). Aborting.", graph.nodeCount(), graph.edgeCount(), maxNodesAndEdges));
- aborted = true;
- break;
- }
-
- reporter.progress();
- }
- }
-
- @Override
- public DataBag getValue()
- {
- if (aborted)
- {
- return null;
- }
-
- System.out.println(String.format("Nodes: %d, Edges: %d", graph.nodeCount(), graph.edgeCount()));
-
- ProgressIndicator progressIndicator = getProgressIndicator();
- System.out.println("Finished loading graph.");
- long startTime = System.nanoTime();
- System.out.println("Initializing.");
- try
- {
- graph.init(progressIndicator);
- }
- catch (IOException e)
- {
- e.printStackTrace();
- return null;
- }
- System.out.println(String.format("Done, took %f ms", (System.nanoTime() - startTime)/10.0e6));
-
- float totalDiff;
- int iter = 0;
-
- System.out.println("Beginning iterations");
- startTime = System.nanoTime();
- do
- {
- // TODO log percentage complete every 5 minutes
- try
- {
- totalDiff = graph.nextIteration(progressIndicator);
- }
- catch (IOException e)
- {
- e.printStackTrace();
- return null;
- }
- iter++;
- } while(iter < maxIters && totalDiff > tolerance);
- System.out.println(String.format("Done, %d iterations took %f ms", iter, (System.nanoTime() - startTime)/10.0e6));
-
- DataBag output = bagFactory.newDefaultBag();
-
- for (Int2IntMap.Entry node : graph.getNodeIds())
- {
- int nodeId = node.getIntKey();
- float rank = graph.getNodeRank(nodeId);
- List nodeData = new ArrayList(2);
- nodeData.add(nodeId);
- nodeData.add(rank);
- output.add(tupleFactory.newTuple(nodeData));
- }
-
- return output;
- }
-
- @Override
- public void cleanup()
- {
- try
- {
- aborted = false;
- this.graph.clear();
- }
- catch (IOException e)
- {
- e.printStackTrace();
- }
- }
-
- private ProgressIndicator getProgressIndicator()
- {
- return new ProgressIndicator()
- {
- @Override
- public void progress()
- {
- reporter.progress();
- }
- };
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try
- {
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.BAG)
- {
- throw new RuntimeException("Expected a BAG as input");
- }
-
- Schema inputBagSchema = inputFieldSchema.schema;
-
- if (inputBagSchema.getField(0).type != DataType.TUPLE)
- {
- throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
- DataType.findTypeName(inputBagSchema.getField(0).type)));
- }
-
- Schema inputTupleSchema = inputBagSchema.getField(0).schema;
-
- if (!this.enableNodeBiasing)
- {
- if (inputTupleSchema.size() != 2)
- {
- throw new RuntimeException("Expected two fields for the node data");
- }
- }
- else
- {
- if (inputTupleSchema.size() != 3)
- {
- throw new RuntimeException("Expected three fields for the node data");
- }
- }
-
- if (inputTupleSchema.getField(0).type != DataType.INTEGER)
- {
- throw new RuntimeException(String.format("Expected source to be an INTEGER, but instead found %s",
- DataType.findTypeName(inputTupleSchema.getField(0).type)));
- }
-
- if (inputTupleSchema.getField(1).type != DataType.BAG)
- {
- throw new RuntimeException(String.format("Expected edges to be represented with a BAG"));
- }
-
- if (this.enableNodeBiasing && inputTupleSchema.getField(2).type != DataType.DOUBLE)
- {
- throw new RuntimeException(String.format("Expected node bias to be a DOUBLE, but instead found %s",
- DataType.findTypeName(inputTupleSchema.getField(2).type)));
- }
-
- Schema.FieldSchema edgesFieldSchema = inputTupleSchema.getField(1);
-
- if (edgesFieldSchema.schema.getField(0).type != DataType.TUPLE)
- {
- throw new RuntimeException(String.format("Expected edges field to contain a TUPLE, but instead found %s",
- DataType.findTypeName(edgesFieldSchema.schema.getField(0).type)));
- }
-
- Schema edgesTupleSchema = edgesFieldSchema.schema.getField(0).schema;
-
- if (edgesTupleSchema.size() != 2)
- {
- throw new RuntimeException("Expected two fields for the edge data");
- }
-
- if (edgesTupleSchema.getField(0).type != DataType.INTEGER)
- {
- throw new RuntimeException(String.format("Expected destination edge ID to an INTEGER, but instead found %s",
- DataType.findTypeName(edgesTupleSchema.getField(0).type)));
- }
-
- if (edgesTupleSchema.getField(1).type != DataType.DOUBLE)
- {
- throw new RuntimeException(String.format("Expected destination edge weight to a DOUBLE, but instead found %s",
- DataType.findTypeName(edgesTupleSchema.getField(1).type)));
- }
-
- Schema tupleSchema = new Schema();
- tupleSchema.add(new Schema.FieldSchema("node",DataType.INTEGER));
- tupleSchema.add(new Schema.FieldSchema("rank",DataType.FLOAT));
-
- return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
- .getName()
- .toLowerCase(), input),
- tupleSchema,
- DataType.BAG));
- }
- catch (FrontendException e)
- {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/linkanalysis/PageRankImpl.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/linkanalysis/PageRankImpl.java b/src/java/datafu/pig/linkanalysis/PageRankImpl.java
deleted file mode 100644
index 5d0b932..0000000
--- a/src/java/datafu/pig/linkanalysis/PageRankImpl.java
+++ /dev/null
@@ -1,571 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.linkanalysis;
-
-import it.unimi.dsi.fastutil.floats.FloatArrayList;
-import it.unimi.dsi.fastutil.ints.Int2IntMap;
-import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
-import it.unimi.dsi.fastutil.ints.IntArrayList;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.Map;
-
-import com.google.common.collect.AbstractIterator;
-
-/**
- * An implementation of {@link <a href="http://en.wikipedia.org/wiki/PageRank" target="_blank">PageRank</a>}, used by the {@link PageRank} UDF.
- * It is not intended to be used directly.
- * </p>
- */
-public class PageRankImpl
-{
- private float totalRankChange;
- private long edgeCount;
- private long nodeCount;
-
- // the damping factor
- private float alpha = 0.85f;
-
- // edge weights (which are doubles) are multiplied by this value so they can be stored as integers internally
- private static float EDGE_WEIGHT_MULTIPLIER = 100000;
-
- private final Int2IntOpenHashMap nodeIndices = new Int2IntOpenHashMap();
- private final FloatArrayList nodeData = new FloatArrayList(); // rank, total weight, contribution, bias(optional), (repeat)
- private int nodeFieldCount = 3; // unless biasing is enabled
-
- private final IntArrayList danglingNodes = new IntArrayList();
-
- private final IntArrayList edges = new IntArrayList(); // source, dest node count... dest id, weight pos, (repeat)
-
- private boolean shouldHandleDanglingNodes = false;
- private boolean shouldCacheEdgesOnDisk = false;
- private long edgeCachingThreshold;
- private boolean nodeBiasingEnabled = false;
-
- private File edgesFile;
- private DataOutputStream edgeDataOutputStream;
- private boolean usingEdgeDiskCache;
-
- public void clear() throws IOException
- {
- this.edgeCount = 0;
- this.nodeCount = 0;
- this.totalRankChange = 0.0f;
-
- this.nodeIndices.clear();
- this.nodeData.clear();
- this.edges.clear();
- this.danglingNodes.clear();
-
- if (edgeDataOutputStream != null)
- {
- this.edgeDataOutputStream.close();
- this.edgeDataOutputStream = null;
- }
-
- this.usingEdgeDiskCache = false;
- this.edgesFile = null;
- }
-
- /**
- * Gets the page rank alpha value.
- * @return alpha
- */
- public float getAlpha()
- {
- return alpha;
- }
-
- /**
- * Sets the page rank alpha value (default is 0.85);
- * @param alpha
- */
- public void setAlpha(float alpha)
- {
- this.alpha = alpha;
- }
-
- public boolean isNodeBiasingEnabled()
- {
- return this.nodeBiasingEnabled;
- }
-
- public void enableNodeBiasing()
- {
- this.nodeBiasingEnabled = true;
- this.nodeFieldCount = 4;
- }
-
- public void disableNodeBiasing()
- {
- this.nodeBiasingEnabled = false;
- this.nodeFieldCount = 3;
- }
-
-
- /**
- * Gets whether disk is being used to cache edges.
- * @return True if the edges are cached on disk.
- */
- public boolean isUsingEdgeDiskCache()
- {
- return usingEdgeDiskCache;
- }
-
- /**
- * Enable disk caching of edges once there are too many (disabled by default).
- */
- public void enableEdgeDiskCaching()
- {
- shouldCacheEdgesOnDisk = true;
- }
-
- /**
- * Disable disk caching of edges once there are too many (disabled by default).
- */
- public void disableEdgeDiskCaching()
- {
- shouldCacheEdgesOnDisk = false;
- }
-
- /**
- * Gets whether edge disk caching is enabled.
- * @return True if edge disk caching is enabled.
- */
- public boolean isEdgeDiskCachingEnabled()
- {
- return shouldCacheEdgesOnDisk;
- }
-
- /**
- * Gets the number of edges past which they will be cached on disk instead of in memory.
- * Edge disk caching must be enabled for this to have any effect.
- * @return Edge count past which caching occurs
- */
- public long getEdgeCachingThreshold()
- {
- return edgeCachingThreshold;
- }
-
- /**
- * Set the number of edges past which they will be cached on disk instead of in memory.
- * Edge disk caching must be enabled for this to have any effect.
- * @param count Edge count past which caching occurs
- */
- public void setEdgeCachingThreshold(long count)
- {
- edgeCachingThreshold = count;
- }
-
- /**
- * Enables dangling node handling (disabled by default).
- */
- public void enableDanglingNodeHandling()
- {
- shouldHandleDanglingNodes = true;
- }
-
- /**
- * Disables dangling node handling (disabled by default).
- */
- public void disableDanglingNodeHandling()
- {
- shouldHandleDanglingNodes = false;
- }
-
- public long nodeCount()
- {
- return this.nodeCount;
- }
-
- public long edgeCount()
- {
- return this.edgeCount;
- }
-
- public Int2IntMap.FastEntrySet getNodeIds()
- {
- return this.nodeIndices.int2IntEntrySet();
- }
-
- public float getNodeRank(int nodeId)
- {
- int nodeIndex = this.nodeIndices.get(nodeId);
- return nodeData.get(nodeIndex);
- }
-
- public float getTotalRankChange()
- {
- return this.totalRankChange;
- }
-
- private void maybeCreateNode(int nodeId)
- {
- // create from node if it doesn't already exist
- if (!nodeIndices.containsKey(nodeId))
- {
- int index = this.nodeData.size();
-
- this.nodeData.add(0.0f); // rank
- this.nodeData.add(0.0f); // total weight
- this.nodeData.add(0.0f); // contribution
-
- if (this.nodeBiasingEnabled)
- {
- this.nodeData.add(0.0f); // bias
- }
-
- this.nodeIndices.put(nodeId, index);
-
- this.nodeCount++;
- }
- }
-
- public float getNodeBias(int nodeId)
- {
- if (!this.nodeBiasingEnabled)
- {
- throw new IllegalArgumentException("Node biasing not enable");
- }
- int nodeIndex = this.nodeIndices.get(nodeId);
- return this.nodeData.get(nodeIndex+3);
- }
-
- public void setNodeBias(int nodeId, float bias)
- {
- if (!this.nodeBiasingEnabled)
- {
- throw new IllegalArgumentException("Node biasing not enable");
- }
-
- int nodeIndex = this.nodeIndices.get(nodeId);
- this.nodeData.set(nodeIndex+3, bias);
- }
-
- public void addNode(Integer sourceId, ArrayList<Map<String,Object>> sourceEdges) throws IOException
- {
- // with bias of 1.0, all nodes have an equal bias (that is, no bias)
- addNode(sourceId, sourceEdges, 1.0f);
- }
-
- public void addNode(Integer sourceId, ArrayList<Map<String,Object>> sourceEdges, float bias) throws IOException
- {
- int source = sourceId.intValue();
-
- maybeCreateNode(source);
-
- if (this.nodeBiasingEnabled)
- {
- setNodeBias(source, bias);
- }
- else if (bias != 1.0f)
- {
- // with node biasing disabled, all nodes implicitly have a bias of 1.0, which means no bias, so if anything else was specified
- // it won't take effect.
- throw new IllegalArgumentException("Bias was specified but node biasing not enabled");
- }
-
- if (this.shouldCacheEdgesOnDisk && !usingEdgeDiskCache && (sourceEdges.size() + this.edgeCount) >= this.edgeCachingThreshold)
- {
- writeEdgesToDisk();
- }
-
- // store the source node id itself
- appendEdgeData(source);
-
- // store how many outgoing edges this node has
- appendEdgeData(sourceEdges.size());
-
- // store the outgoing edges
- for (Map<String,Object> edge : sourceEdges)
- {
- int dest = ((Integer)edge.get("dest")).intValue();
- float weight = ((Double)edge.get("weight")).floatValue();
-
- maybeCreateNode(dest);
-
- appendEdgeData(dest);
-
- // location of weight in weights array
- appendEdgeData(Math.max(1, (int)(weight * EDGE_WEIGHT_MULTIPLIER)));
-
- this.edgeCount++;
- }
- }
-
- private void appendEdgeData(int data) throws IOException
- {
- if (this.edgeDataOutputStream != null)
- {
- this.edgeDataOutputStream.writeInt(data);
- }
- else
- {
- this.edges.add(data);
- }
- }
-
- public void init() throws IOException
- {
- init(getDummyIndicator());
- }
-
- public void init(ProgressIndicator progressIndicator) throws IOException
- {
- if (this.edgeDataOutputStream != null)
- {
- this.edgeDataOutputStream.close();
- this.edgeDataOutputStream = null;
- }
-
- // initialize all nodes to an equal share of the total rank (1.0)
- float nodeRank = 1.0f / this.nodeCount;
- float totalBias = 0.0f;
- for (int j=0; j<this.nodeData.size(); j+=this.nodeFieldCount)
- {
- nodeData.set(j, nodeRank);
- progressIndicator.progress();
- if (this.nodeBiasingEnabled)
- {
- totalBias += nodeData.getFloat(j+3);
- }
- }
-
- // if node biasing enabled, need to normalize the bias by the total bias across all nodes so it represents
- // the share of bias.
- if (this.nodeBiasingEnabled)
- {
- for (int j=0; j<this.nodeData.size(); j+=this.nodeFieldCount)
- {
- float bias = nodeData.getFloat(j+3);
- bias /= totalBias;
- nodeData.set(j+3,bias);
- }
- }
-
- Iterator<Integer> edgeData = getEdgeData();
-
- while(edgeData.hasNext())
- {
- int sourceId = edgeData.next();
- int nodeEdgeCount = edgeData.next();
-
- while (nodeEdgeCount-- > 0)
- {
- // skip the destination node id
- edgeData.next();
-
- float weight = edgeData.next();
-
- int nodeIndex = this.nodeIndices.get(sourceId);
-
- float totalWeight = this.nodeData.getFloat(nodeIndex+1);
- totalWeight += weight;
- this.nodeData.set(nodeIndex+1, totalWeight);
-
- progressIndicator.progress();
- }
- }
-
- // if handling dangling nodes, get a list of them by finding those nodes with no outgoing
- // edges (i.e. total outgoing edge weight is 0.0)
- if (shouldHandleDanglingNodes)
- {
- for (Map.Entry<Integer,Integer> e : nodeIndices.entrySet())
- {
- int nodeId = e.getKey();
- int nodeIndex = e.getValue();
- float totalWeight = nodeData.getFloat(nodeIndex+1);
- if (totalWeight == 0.0f)
- {
- danglingNodes.add(nodeId);
- }
- }
- }
- }
-
- public float nextIteration(ProgressIndicator progressIndicator) throws IOException
- {
- distribute(progressIndicator);
- commit(progressIndicator);
-
- return getTotalRankChange();
- }
-
- public float nextIteration() throws IOException
- {
- ProgressIndicator dummyIndicator = getDummyIndicator();
- distribute(dummyIndicator);
- commit(dummyIndicator);
-
- return getTotalRankChange();
- }
-
- private ProgressIndicator getDummyIndicator()
- {
- return new ProgressIndicator() {
- @Override
- public void progress()
- {
- }
- };
- }
-
- public void distribute(ProgressIndicator progressIndicator) throws IOException
- {
- Iterator<Integer> edgeData = getEdgeData();
-
- while(edgeData.hasNext())
- {
- int sourceId = edgeData.next();
- int nodeEdgeCount = edgeData.next();
-
- while (nodeEdgeCount-- > 0)
- {
- int toId = edgeData.next();
- float weight = edgeData.next();
-
- int fromNodeIndex = this.nodeIndices.get(sourceId);
- int toNodeIndex = this.nodeIndices.get(toId);
-
- float contributionChange = weight * this.nodeData.getFloat(fromNodeIndex) / this.nodeData.getFloat(fromNodeIndex+1);
-
- float currentContribution = this.nodeData.getFloat(toNodeIndex+2);
- this.nodeData.set(toNodeIndex+2, currentContribution + contributionChange);
-
- progressIndicator.progress();
- }
- }
-
- if (shouldHandleDanglingNodes)
- {
- // get the rank from each of the dangling nodes
- float totalRank = 0.0f;
- for (int nodeId : danglingNodes)
- {
- int nodeIndex = nodeIndices.get(nodeId);
- float rank = nodeData.get(nodeIndex);
- totalRank += rank;
- }
-
- // distribute the dangling node ranks to all the nodes in the graph
- // note: the alpha factor is applied in the commit stage
- float contributionIncrease = totalRank / this.nodeCount;
- for (int i=2; i<nodeData.size(); i += this.nodeFieldCount)
- {
- float contribution = nodeData.getFloat(i);
- contribution += contributionIncrease;
- nodeData.set(i, contribution);
- }
- }
- }
-
- public void commit(ProgressIndicator progressIndicator)
- {
- this.totalRankChange = 0.0f;
-
- float oneMinusAlpha = (1.0f - this.alpha);
- float oneMinusAlphaOverNodeCount = oneMinusAlpha / nodeCount;
-
- for (int nodeIndex=0; nodeIndex<this.nodeData.size(); nodeIndex += this.nodeFieldCount)
- {
- float oldRank = this.nodeData.get(nodeIndex+2);
- float newRank;
-
- if (this.nodeBiasingEnabled)
- {
- float bias = this.nodeData.get(nodeIndex+3);
- newRank = bias * oneMinusAlpha + alpha * oldRank;
- }
- else
- {
- newRank = oneMinusAlphaOverNodeCount + alpha * oldRank;
- }
-
- this.nodeData.set(nodeIndex+2, 0.0f);
-
- float lastRankDiff = newRank - this.nodeData.get(nodeIndex);
-
- this.nodeData.set(nodeIndex, newRank);
-
- this.totalRankChange += Math.abs(lastRankDiff);
-
- progressIndicator.progress();
- }
- }
-
- private void writeEdgesToDisk() throws IOException
- {
- this.edgesFile = File.createTempFile("fastgraph", null);
-
- FileOutputStream outStream = new FileOutputStream(this.edgesFile);
- BufferedOutputStream bufferedStream = new BufferedOutputStream(outStream);
- this.edgeDataOutputStream = new DataOutputStream(bufferedStream);
-
- for (int edgeData : edges)
- {
- this.edgeDataOutputStream.writeInt(edgeData);
- }
-
- this.edges.clear();
- usingEdgeDiskCache = true;
- }
-
- private Iterator<Integer> getEdgeData() throws IOException
- {
- if (!usingEdgeDiskCache)
- {
- return this.edges.iterator();
- }
- else
- {
- FileInputStream fileInputStream = new FileInputStream(this.edgesFile);
- BufferedInputStream inputStream = new BufferedInputStream(fileInputStream);
- final DataInputStream dataInputStream = new DataInputStream(inputStream);
-
- return new AbstractIterator<Integer>() {
-
- @Override
- protected Integer computeNext()
- {
- try
- {
- return dataInputStream.readInt();
- }
- catch (IOException e)
- {
- return endOfData();
- }
- }
-
- };
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/linkanalysis/ProgressIndicator.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/linkanalysis/ProgressIndicator.java b/src/java/datafu/pig/linkanalysis/ProgressIndicator.java
deleted file mode 100644
index 3dc54c5..0000000
--- a/src/java/datafu/pig/linkanalysis/ProgressIndicator.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.linkanalysis;
-
-interface ProgressIndicator
-{
- void progress();
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/linkanalysis/package-info.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/linkanalysis/package-info.java b/src/java/datafu/pig/linkanalysis/package-info.java
deleted file mode 100644
index 2c6c078..0000000
--- a/src/java/datafu/pig/linkanalysis/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * UDFs for performing link analysis, such as PageRank.
- */
-package datafu.pig.linkanalysis;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/random/RandInt.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/random/RandInt.java b/src/java/datafu/pig/random/RandInt.java
deleted file mode 100644
index de89c4a..0000000
--- a/src/java/datafu/pig/random/RandInt.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.random;
-
-import java.io.IOException;
-import java.util.Random;
-
-import org.apache.pig.builtin.Nondeterministic;
-import org.apache.pig.data.DataType;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-import datafu.pig.util.SimpleEvalFunc;
-
-/**
- * Generates a uniformly distributed integer between two bounds.
- */
-@Nondeterministic
-public class RandInt extends SimpleEvalFunc<Integer>
-{
- private final Random rand = new Random();
-
- /**
- * @param min lower bound for random number
- * @param max upper bound for random number
- */
- public Integer call(Integer min, Integer max) throws IOException
- {
- try
- {
- if (min > max)
- {
- throw new RuntimeException("The first argument must be less than the second");
- }
- return rand.nextInt(max - min + 1) + min;
- }
- catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- return new Schema(new Schema.FieldSchema("rand", DataType.INTEGER));
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/random/RandomUUID.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/random/RandomUUID.java b/src/java/datafu/pig/random/RandomUUID.java
deleted file mode 100644
index d63f4cf..0000000
--- a/src/java/datafu/pig/random/RandomUUID.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package datafu.pig.random;
-
-import java.io.IOException;
-import java.util.UUID;
-
-import org.apache.pig.EvalFunc;
-import org.apache.pig.builtin.Nondeterministic;
-import org.apache.pig.data.*;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * Generates a random UUID using java.util.UUID
- */
-@Nondeterministic
-public class RandomUUID extends EvalFunc<String>
-{
- public String exec(Tuple input) throws IOException
- {
- return UUID.randomUUID().toString();
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- return new Schema(new Schema.FieldSchema("uuid", DataType.CHARARRAY));
- }
-}
-
[03/19] DATAFU-27 Migrate build system to Gradle
Posted by wv...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/sampling/SimpleRandomSampleTest.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/sampling/SimpleRandomSampleTest.java b/test/pig/datafu/test/pig/sampling/SimpleRandomSampleTest.java
deleted file mode 100644
index bd34881..0000000
--- a/test/pig/datafu/test/pig/sampling/SimpleRandomSampleTest.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.sampling;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.pig.sampling.SimpleRandomSample;
-import datafu.test.pig.PigTests;
-
-/**
- * Tests for {@link SimpleRandomSample}.
- *
- * @author ximeng
- *
- */
-public class SimpleRandomSampleTest extends PigTests
-{
- /**
- * register $JAR_PATH
- *
- * DEFINE SRS datafu.pig.sampling.SimpleRandomSample();
- *
- * data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
- *
- * sampled = FOREACH (GROUP data ALL) GENERATE SRS(data, $p) as sample_data;
- *
- * sampled = FOREACH sampled GENERATE COUNT(sample_data) AS sample_count;
- *
- * STORE sampled INTO 'output';
- */
- @Multiline
- private String simpleRandomSampleTest;
-
- /**
- * register $JAR_PATH
- *
- * DEFINE SRS datafu.pig.sampling.SimpleRandomSample();
- *
- * data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
- *
- * sampled = FOREACH (GROUP data ALL) GENERATE SRS(data, $p, $n1) as sample_data;
- *
- * sampled = FOREACH sampled GENERATE COUNT(sample_data) AS sample_count;
- *
- * STORE sampled INTO 'output';
- */
- @Multiline
- private String simpleRandomSampleWithLowerBoundTest;
-
- /**
- * register $JAR_PATH
- *
- * DEFINE SRS datafu.pig.sampling.SimpleRandomSample();
- *
- * data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
- *
- * sampled = FOREACH (GROUP data ALL) GENERATE SRS(data, $p1) as sample_1, SRS(data,
- * $p2) AS sample_2;
- *
- * sampled = FOREACH sampled GENERATE COUNT(sample_1) AS sample_count_1, COUNT(sample_2)
- * AS sample_count_2;
- *
- * STORE sampled INTO 'output';
- */
- @Multiline
- private String simpleRandomSampleWithTwoCallsTest;
-
- @Test
- public void testSimpleRandomSample() throws Exception
- {
- writeLinesToFile("input",
- "A1\tB1\t1",
- "A1\tB1\t4",
- "A1\tB3\t4",
- "A1\tB4\t4",
- "A2\tB1\t4",
- "A2\tB2\t4",
- "A3\tB1\t3",
- "A3\tB1\t1",
- "A3\tB3\t77",
- "A4\tB1\t3",
- "A4\tB2\t3",
- "A4\tB3\t59",
- "A4\tB4\t29",
- "A5\tB1\t4",
- "A6\tB2\t3",
- "A6\tB2\t55",
- "A6\tB3\t1",
- "A7\tB1\t39",
- "A7\tB2\t27",
- "A7\tB3\t85",
- "A8\tB1\t4",
- "A8\tB2\t45",
- "A9\tB3\t92",
- "A9\tB3\t0",
- "A9\tB6\t42",
- "A9\tB5\t1",
- "A10\tB1\t7",
- "A10\tB2\t23",
- "A10\tB2\t1",
- "A10\tB2\t31",
- "A10\tB6\t41",
- "A10\tB7\t52");
-
- int n = 32;
- double p = 0.3;
- int s = (int) Math.ceil(p * n);
-
- PigTest test = createPigTestFromString(simpleRandomSampleTest, "p=" + p);
- test.runScript();
- assertOutput(test, "sampled", "(" + s + ")");
-
- int n1 = 30;
- PigTest testWithLB =
- createPigTestFromString(simpleRandomSampleWithLowerBoundTest, "p=" + p, "n1="
- + n1);
- testWithLB.runScript();
- assertOutput(testWithLB, "sampled", "(" + s + ")");
-
- double p1 = 0.05;
- double p2 = 0.95;
- int s1 = (int) Math.ceil(p1 * n);
- int s2 = (int) Math.ceil(p2 * n);
-
- PigTest testWithTwoCalls =
- createPigTestFromString(simpleRandomSampleWithTwoCallsTest, "p1=" + p1, "p2="
- + p2);
- testWithTwoCalls.runScript();
- assertOutput(testWithTwoCalls, "sampled", "(" + s1 + "," + s2 + ")");
-
- test.runScript();
-
- }
-
- /**
- * register $JAR_PATH
- *
- * DEFINE SRS datafu.pig.sampling.SimpleRandomSample();
- *
- * data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
- *
- * sampled = FOREACH (GROUP data BY A_id) GENERATE group, SRS(data,
- * $SAMPLING_PROBABILITY) as sample_data;
- *
- * sampled = FOREACH sampled GENERATE group, COUNT(sample_data) AS sample_count;
- *
- * sampled = ORDER sampled BY group;
- *
- * STORE sampled INTO 'output';
- */
- @Multiline
- private String stratifiedSampleTest;
-
- @Test
- public void testStratifiedSample() throws Exception
- {
- writeLinesToFile("input",
- "A1\tB1\t1",
- "A1\tB1\t4",
- "A1\tB3\t4",
- "A1\tB4\t4",
- "A2\tB1\t4",
- "A2\tB2\t4",
- "A3\tB1\t3",
- "A3\tB1\t1",
- "A3\tB3\t77",
- "A4\tB1\t3",
- "A4\tB2\t3",
- "A4\tB3\t59",
- "A4\tB4\t29",
- "A5\tB1\t4",
- "A6\tB2\t3",
- "A6\tB2\t55",
- "A6\tB3\t1",
- "A7\tB1\t39",
- "A7\tB2\t27",
- "A7\tB3\t85",
- "A8\tB1\t4",
- "A8\tB2\t45",
- "A9\tB3\t92",
- "A9\tB3\t0",
- "A9\tB6\t42",
- "A9\tB5\t1",
- "A10\tB1\t7",
- "A10\tB2\t23",
- "A10\tB2\t1",
- "A10\tB2\t31",
- "A10\tB6\t41",
- "A10\tB7\t52");
-
- double p = 0.5;
-
- PigTest test =
- createPigTestFromString(stratifiedSampleTest, "SAMPLING_PROBABILITY=" + p);
- test.runScript();
- assertOutput(test,
- "sampled",
- "(A1,2)",
- "(A10,3)",
- "(A2,1)",
- "(A3,2)",
- "(A4,2)",
- "(A5,1)",
- "(A6,2)",
- "(A7,2)",
- "(A8,1)",
- "(A9,2)");
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/sampling/SimpleRandomSampleTestOld.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/sampling/SimpleRandomSampleTestOld.java b/test/pig/datafu/test/pig/sampling/SimpleRandomSampleTestOld.java
deleted file mode 100644
index 15e1fd6..0000000
--- a/test/pig/datafu/test/pig/sampling/SimpleRandomSampleTestOld.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.sampling;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.pig.sampling.SimpleRandomSample;
-import datafu.test.pig.PigTests;
-
-/**
- * Tests for {@link SimpleRandomSample}.
- *
- * @deprecated This tests the deprecated functionality of SimpleRandomSample
- * where the probability can be specified in the constructor.
- * @author ximeng
- *
- */
-public class SimpleRandomSampleTestOld extends PigTests
-{
- /**
- * register $JAR_PATH
- *
- * DEFINE SRS datafu.pig.sampling.SimpleRandomSample('$SAMPLING_PROBABILITY');
- *
- * data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
- *
- * sampled = FOREACH (GROUP data ALL) GENERATE SRS(data) as sample_data;
- *
- * sampled = FOREACH sampled GENERATE COUNT(sample_data) AS sample_count;
- *
- * STORE sampled INTO 'output';
- */
- @Multiline
- private String simpleRandomSampleTest;
-
- @Test
- public void simpleRandomSampleTest() throws Exception
- {
- writeLinesToFile("input",
- "A1\tB1\t1",
- "A1\tB1\t4",
- "A1\tB3\t4",
- "A1\tB4\t4",
- "A2\tB1\t4",
- "A2\tB2\t4",
- "A3\tB1\t3",
- "A3\tB1\t1",
- "A3\tB3\t77",
- "A4\tB1\t3",
- "A4\tB2\t3",
- "A4\tB3\t59",
- "A4\tB4\t29",
- "A5\tB1\t4",
- "A6\tB2\t3",
- "A6\tB2\t55",
- "A6\tB3\t1",
- "A7\tB1\t39",
- "A7\tB2\t27",
- "A7\tB3\t85",
- "A8\tB1\t4",
- "A8\tB2\t45",
- "A9\tB3\t92",
- "A9\tB3\t0",
- "A9\tB6\t42",
- "A9\tB5\t1",
- "A10\tB1\t7",
- "A10\tB2\t23",
- "A10\tB2\t1",
- "A10\tB2\t31",
- "A10\tB6\t41",
- "A10\tB7\t52");
-
- int n = 32;
- double p = 0.3;
- int s = (int) Math.ceil(p * n);
- PigTest test =
- createPigTestFromString(simpleRandomSampleTest, "SAMPLING_PROBABILITY=" + p);
-
- test.runScript();
-
- assertOutput(test, "sampled", "(" + s + ")");
- }
-
- /**
- * register $JAR_PATH
- *
- * DEFINE SRS datafu.pig.sampling.SimpleRandomSample('$SAMPLING_PROBABILITY');
- *
- * data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
- *
- * sampled = FOREACH (GROUP data BY A_id) GENERATE group, SRS(data) as sample_data;
- *
- * sampled = FOREACH sampled GENERATE group, COUNT(sample_data) AS sample_count;
- *
- * sampled = ORDER sampled BY group;
- *
- * STORE sampled INTO 'output';
- */
- @Multiline
- private String stratifiedSampleTest;
-
- @Test
- public void stratifiedSampleTest() throws Exception
- {
- writeLinesToFile("input",
- "A1\tB1\t1",
- "A1\tB1\t4",
- "A1\tB3\t4",
- "A1\tB4\t4",
- "A2\tB1\t4",
- "A2\tB2\t4",
- "A3\tB1\t3",
- "A3\tB1\t1",
- "A3\tB3\t77",
- "A4\tB1\t3",
- "A4\tB2\t3",
- "A4\tB3\t59",
- "A4\tB4\t29",
- "A5\tB1\t4",
- "A6\tB2\t3",
- "A6\tB2\t55",
- "A6\tB3\t1",
- "A7\tB1\t39",
- "A7\tB2\t27",
- "A7\tB3\t85",
- "A8\tB1\t4",
- "A8\tB2\t45",
- "A9\tB3\t92",
- "A9\tB3\t0",
- "A9\tB6\t42",
- "A9\tB5\t1",
- "A10\tB1\t7",
- "A10\tB2\t23",
- "A10\tB2\t1",
- "A10\tB2\t31",
- "A10\tB6\t41",
- "A10\tB7\t52");
-
- double p = 0.5;
-
- PigTest test =
- createPigTestFromString(stratifiedSampleTest, "SAMPLING_PROBABILITY=" + p);
- test.runScript();
- assertOutput(test,
- "sampled",
- "(A1,2)",
- "(A10,3)",
- "(A2,1)",
- "(A3,2)",
- "(A4,2)",
- "(A5,1)",
- "(A6,2)",
- "(A7,2)",
- "(A8,1)",
- "(A9,2)");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/sampling/SimpleRandomSampleWithReplacementTest.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/sampling/SimpleRandomSampleWithReplacementTest.java b/test/pig/datafu/test/pig/sampling/SimpleRandomSampleWithReplacementTest.java
deleted file mode 100644
index 338b3e7..0000000
--- a/test/pig/datafu/test/pig/sampling/SimpleRandomSampleWithReplacementTest.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.sampling;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.pig.sampling.SimpleRandomSampleWithReplacementElect;
-import datafu.pig.sampling.SimpleRandomSampleWithReplacementVote;
-import datafu.test.pig.PigTests;
-
-/**
- * Tests for {@link SimpleRandomSampleWithReplacementVote} and
- * {@link SimpleRandomSampleWithReplacementElect}.
- *
- * @author ximeng
- *
- */
-public class SimpleRandomSampleWithReplacementTest extends PigTests
-{
- // @formatter:off
- /**
- * register $JAR_PATH
- *
- * DEFINE SRSWR_VOTE datafu.pig.sampling.SimpleRandomSampleWithReplacementVote();
- *
- * DEFINE SRSWR_ELECT datafu.pig.sampling.SimpleRandomSampleWithReplacementElect();
- *
- * item = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
- *
- * summary = FOREACH (GROUP item ALL) GENERATE COUNT(item) AS count;
- *
- * candidates = FOREACH (GROUP item ALL) GENERATE FLATTEN(SRSWR_VOTE(item, $SAMPLE_SIZE, summary.count));
- *
- * sampled = FOREACH (GROUP candidates BY position) GENERATE FLATTEN(SRSWR_ELECT(candidates));
- *
- * sampled = FOREACH (GROUP sampled ALL) GENERATE COUNT(sampled) AS size;
- *
- * STORE sampled INTO 'output';
- */
- @Multiline
- private String simpleRandomSampleWithReplacementTest;
- // @formatter:on
-
- @Test
- public void testSimpleRandomSampleWithReplacement() throws Exception
- {
- writeLinesToFile("input",
- "A1\tB1\t1",
- "A1\tB1\t4",
- "A1\tB3\t4",
- "A1\tB4\t4",
- "A2\tB1\t4",
- "A2\tB2\t4",
- "A3\tB1\t3",
- "A3\tB1\t1",
- "A3\tB3\t77",
- "A4\tB1\t3",
- "A4\tB2\t3",
- "A4\tB3\t59",
- "A4\tB4\t29",
- "A5\tB1\t4",
- "A6\tB2\t3",
- "A6\tB2\t55",
- "A6\tB3\t1",
- "A7\tB1\t39",
- "A7\tB2\t27",
- "A7\tB3\t85",
- "A8\tB1\t4",
- "A8\tB2\t45",
- "A9\tB3\t92",
- "A9\tB3\t0",
- "A9\tB6\t42",
- "A9\tB5\t1",
- "A10\tB1\t7",
- "A10\tB2\t23",
- "A10\tB2\t1",
- "A10\tB2\t31",
- "A10\tB6\t41",
- "A10\tB7\t52");
-
- int s = 32;
- PigTest test =
- createPigTestFromString(simpleRandomSampleWithReplacementTest, "SAMPLE_SIZE=" + s
- + "L");
-
- test.runScript();
-
- assertOutput(test, "sampled", "(" + s + ")");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/sampling/WeightedReservoirSamplingTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/sampling/WeightedReservoirSamplingTests.java b/test/pig/datafu/test/pig/sampling/WeightedReservoirSamplingTests.java
deleted file mode 100644
index 9bab8a0..0000000
--- a/test/pig/datafu/test/pig/sampling/WeightedReservoirSamplingTests.java
+++ /dev/null
@@ -1,373 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.sampling;
-
-import java.io.IOException;
-import java.util.*;
-
-import junit.framework.Assert;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.pig.sampling.WeightedReservoirSample;
-import datafu.test.pig.PigTests;
-
-/**
- * Tests for {@link WeightedReservoirSample}.
- *
- * @author wjian
- *
- */
-public class WeightedReservoirSamplingTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- DEFINE ReservoirSample datafu.pig.sampling.WeightedReservoirSample('$RESERVOIR_SIZE','2');
- DEFINE Assert datafu.pig.util.Assert();
-
- data = LOAD 'input' AS (A_id:int, B_id:chararray, C:int, W:double);
- sampled = FOREACH (GROUP data BY A_id) GENERATE group as A_id, ReservoirSample(data.(B_id,C,W)) as sample_data;
- sampled = FILTER sampled BY Assert((SIZE(sample_data) <= $RESERVOIR_SIZE ? 1 : 0), 'must be <= $RESERVOIR_SIZE');
- sampled = FOREACH sampled GENERATE A_id, FLATTEN(sample_data);
- STORE sampled INTO 'output';
-
- */
- @Multiline
- private String weightedReservoirSampleGroupTest;
-
- /**
- * Verifies that WeightedReservoirSample works when data grouped by a key.
- * In particular it ensures that the reservoir is not reused across keys.
- *
- * <p>
- * This confirms the fix for DATAFU-11.
- * </p>
- *
- * @throws Exception
- */
- @Test
- public void weightedReservoirSampleGroupTest() throws Exception
- {
- // first value is the key. second to last value matches the key so we can
- // verify the register is reset for each key. values should not
- // bleed across to other keys.
- writeLinesToFile("input",
- "1\tB1\t1\t1.0",
- "1\tB1\t1\t1.0",
- "1\tB3\t1\t1.0",
- "1\tB4\t1\t1.0",
- "2\tB1\t2\t1.0",
- "2\tB2\t2\t1.0",
- "3\tB1\t3\t1.0",
- "3\tB1\t3\t1.0",
- "3\tB3\t3\t1.0",
- "4\tB1\t4\t1.0",
- "4\tB2\t4\t1.0",
- "4\tB3\t4\t1.0",
- "4\tB4\t4\t1.0",
- "5\tB1\t5\t1.0",
- "6\tB2\t6\t1.0",
- "6\tB2\t6\t1.0",
- "6\tB3\t6\t1.0",
- "7\tB1\t7\t1.0",
- "7\tB2\t7\t1.0",
- "7\tB3\t7\t1.0",
- "8\tB1\t8\t1.0",
- "8\tB2\t8\t1.0",
- "9\tB3\t9\t1.0",
- "9\tB3\t9\t1.0",
- "9\tB6\t9\t1.0",
- "9\tB5\t9\t1.0",
- "10\tB1\t10\t1.0",
- "10\tB2\t10\t1.0",
- "10\tB2\t10\t1.0",
- "10\tB2\t10\t1.0",
- "10\tB6\t10\t1.0",
- "10\tB7\t10\t1.0");
-
- for(int i=1; i<=3; i++) {
- int reservoirSize = i ;
- PigTest test = createPigTestFromString(weightedReservoirSampleGroupTest, "RESERVOIR_SIZE="+reservoirSize);
- test.runScript();
-
- List<Tuple> tuples = getLinesForAlias(test, "sampled");
-
- for (Tuple tuple : tuples)
- {
- Assert.assertEquals(((Number)tuple.get(0)).intValue(), ((Number)tuple.get(2)).intValue());
- }
- }
- }
-
- /**
- register $JAR_PATH
-
- define WeightedSample datafu.pig.sampling.WeightedReservoirSample('1','1');
-
- data = LOAD 'input' AS (v1:chararray,v2:INT);
- data_g = group data all;
- sampled = FOREACH data_g GENERATE WeightedSample(data);
- --describe sampled;
-
- STORE sampled INTO 'output';
-
- */
- @Multiline
- private String weightedSampleTest;
-
- @Test
- public void weightedSampleTest() throws Exception
- {
- Map<String, Integer> count = new HashMap<String, Integer>();
-
- count.put("a", 0);
- count.put("b", 0);
- count.put("c", 0);
- count.put("d", 0);
-
- PigTest test = createPigTestFromString(weightedSampleTest);
-
- writeLinesToFile("input",
- "a\t100","b\t1","c\t5","d\t2");
-
- for(int i = 0; i < 10; i++) {
- test.runScript();
-
- List<Tuple> output = this.getLinesForAlias(test, "sampled");
-
- Tuple t = output.get(0);
-
- DataBag sampleBag = (DataBag)t.get(0);
-
- for(Iterator<Tuple> sampleIter = sampleBag.iterator(); sampleIter.hasNext();) {
- Tuple st = sampleIter.next();
- String key = (String)st.get(0);
- count.put(key, count.get(key) + 1);
- }
- }
-
- String maxKey = "";
- int maxCount = 0;
- for(String key : count.keySet()) {
- if(maxCount < count.get(key)) {
- maxKey = key;
- maxCount = count.get(key);
- }
- }
-
- Assert.assertEquals(maxKey, "a");
- }
-
- @Test
- public void weightedReservoirSampleAccumulateTest() throws IOException
- {
- WeightedReservoirSample sampler = new WeightedReservoirSample("10", "1");
-
- for (int i=0; i<100; i++)
- {
- Tuple t = TupleFactory.getInstance().newTuple(2);
- t.set(0, i);
- t.set(1, i + 1);
- DataBag bag = BagFactory.getInstance().newDefaultBag();
- bag.add(t);
- Tuple input = TupleFactory.getInstance().newTuple(bag);
- sampler.accumulate(input);
- }
-
- DataBag result = sampler.getValue();
- verifyNoRepeatAllFound(result, 10, 0, 100);
- }
-
- @Test
- public void weightedReservoirSampleAlgebraicTest() throws IOException
- {
- WeightedReservoirSample.Initial initialSampler = new WeightedReservoirSample.Initial("10", "1");
- WeightedReservoirSample.Intermediate intermediateSampler = new WeightedReservoirSample.Intermediate("10", "1");
- WeightedReservoirSample.Final finalSampler = new WeightedReservoirSample.Final("10", "1");
-
- DataBag bag = BagFactory.getInstance().newDefaultBag();
- for (int i=0; i<100; i++)
- {
- Tuple t = TupleFactory.getInstance().newTuple(2);
- t.set(0, i);
- t.set(1, i + 1);
- bag.add(t);
- }
-
- Tuple input = TupleFactory.getInstance().newTuple(bag);
-
- Tuple intermediateTuple = initialSampler.exec(input);
- DataBag intermediateBag = BagFactory.getInstance().newDefaultBag(Arrays.asList(intermediateTuple));
- intermediateTuple = intermediateSampler.exec(TupleFactory.getInstance().newTuple(intermediateBag));
- intermediateBag = BagFactory.getInstance().newDefaultBag(Arrays.asList(intermediateTuple));
- DataBag result = finalSampler.exec(TupleFactory.getInstance().newTuple(intermediateBag));
- verifyNoRepeatAllFound(result, 10, 0, 100);
- }
-
- private void verifyNoRepeatAllFound(DataBag result,
- int expectedResultSize,
- int left,
- int right) throws ExecException
- {
- Assert.assertEquals(expectedResultSize, result.size());
-
- // all must be found, no repeats
- Set<Integer> found = new HashSet<Integer>();
- for (Tuple t : result)
- {
- Integer i = (Integer)t.get(0);
- Assert.assertTrue(i>=left && i<right);
- Assert.assertFalse(String.format("Found duplicate of %d",i), found.contains(i));
- found.add(i);
- }
- }
-
- @Test
- public void weightedReservoirSampleLimitExecTest() throws IOException
- {
- WeightedReservoirSample sampler = new WeightedReservoirSample("100", "1");
-
- DataBag bag = BagFactory.getInstance().newDefaultBag();
- for (int i=0; i<10; i++)
- {
- Tuple t = TupleFactory.getInstance().newTuple(2);
- t.set(0, i);
- t.set(1, 1); // score is equal for all
- bag.add(t);
- }
-
- Tuple input = TupleFactory.getInstance().newTuple(1);
- input.set(0, bag);
-
- DataBag result = sampler.exec(input);
-
- verifyNoRepeatAllFound(result, 10, 0, 10);
-
- Set<Integer> found = new HashSet<Integer>();
- for (Tuple t : result)
- {
- Integer i = (Integer)t.get(0);
- found.add(i);
- }
-
- for(int i = 0; i < 10; i++)
- {
- Assert.assertTrue(found.contains(i));
- }
- }
-
- @Test
- public void invalidConstructorArgTest() throws Exception
- {
- try {
- WeightedReservoirSample sampler = new WeightedReservoirSample("1", "-1");
- Assert.fail( "Testcase should fail");
- } catch (Exception ex) {
- Assert.assertTrue(ex.getMessage().indexOf("Invalid negative index of weight field argument for WeightedReserviorSample constructor: -1") >= 0);
- }
- }
-
- @Test
- public void invalidWeightTest() throws Exception
- {
- PigTest test = createPigTestFromString(weightedSampleTest);
-
- writeLinesToFile("input",
- "a\t100","b\t1","c\t0","d\t2");
- try {
- test.runScript();
- List<Tuple> output = this.getLinesForAlias(test, "sampled");
- Assert.fail( "Testcase should fail");
- } catch (Exception ex) {
- }
- }
-
- /**
- register $JAR_PATH
-
- define WeightedSample datafu.pig.sampling.WeightedReservoirSample('1','1');
-
- data = LOAD 'input' AS (v1:chararray);
- data_g = group data all;
- sampled = FOREACH data_g GENERATE WeightedSample(data);
- describe sampled;
-
- STORE sampled INTO 'output';
-
- */
- @Multiline
- private String invalidInputTupleSizeTest;
-
- @Test
- public void invalidInputTupleSizeTest() throws Exception
- {
- PigTest test = createPigTestFromString(invalidInputTupleSizeTest);
-
- writeLinesToFile("input",
- "a","b","c","d");
- try {
- test.runScript();
- List<Tuple> output = this.getLinesForAlias(test, "sampled");
- Assert.fail( "Testcase should fail");
- } catch (Exception ex) {
- Assert.assertTrue(ex.getMessage().indexOf("The field schema of the input tuple is null or the tuple size is no more than the weight field index: 1") >= 0);
- }
- }
-
- /**
- register $JAR_PATH
-
- define WeightedSample datafu.pig.sampling.WeightedReservoirSample('1','0');
-
- data = LOAD 'input' AS (v1:chararray, v2:INT);
- data_g = group data all;
- sampled = FOREACH data_g GENERATE WeightedSample(data);
- describe sampled;
-
- STORE sampled INTO 'output';
-
- */
- @Multiline
- private String invalidWeightFieldSchemaTest;
-
- @Test
- public void invalidWeightFieldSchemaTest() throws Exception
- {
- PigTest test = createPigTestFromString(invalidWeightFieldSchemaTest);
-
- writeLinesToFile("input",
- "a\t100","b\t1","c\t5","d\t2");
- try {
- test.runScript();
- List<Tuple> output = this.getLinesForAlias(test, "sampled");
- Assert.fail( "Testcase should fail");
- } catch (Exception ex) {
- Assert.assertTrue(ex.getMessage().indexOf("Expect the type of the weight field of the input tuple to be of ([int, long, float, double]), but instead found (chararray), weight field: 0") >= 0);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/sessions/SessionTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/sessions/SessionTests.java b/test/pig/datafu/test/pig/sessions/SessionTests.java
deleted file mode 100644
index 084f970..0000000
--- a/test/pig/datafu/test/pig/sessions/SessionTests.java
+++ /dev/null
@@ -1,433 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.sessions;
-
-import static org.testng.Assert.*;
-
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-
-import junit.framework.Assert;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.commons.lang.StringUtils;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.pigunit.PigTest;
-import org.joda.time.DateTime;
-import org.testng.annotations.Test;
-
-import datafu.pig.sessions.SessionCount;
-import datafu.pig.sessions.Sessionize;
-import datafu.test.pig.PigTests;
-
-public class SessionTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- define Sessionize datafu.pig.sessions.Sessionize('$TIME_WINDOW');
-
- views = LOAD 'input' AS (time:$TIME_TYPE, user_id:int, value:int);
-
- views_grouped = GROUP views BY user_id;
- view_counts = FOREACH views_grouped {
- views = ORDER views BY time;
- GENERATE flatten(Sessionize(views)) as (time,user_id,value,session_id);
- }
-
- max_value = GROUP view_counts BY (user_id, session_id);
-
- max_value = FOREACH max_value GENERATE group.user_id, MAX(view_counts.value) AS val;
-
- STORE max_value INTO 'output';
- */
- @Multiline private String sessionizeTest;
-
- private String[] inputData = new String[] {
- "2010-01-01T01:00:00Z\t1\t10",
- "2010-01-01T01:15:00Z\t1\t20",
- "2010-01-01T01:31:00Z\t1\t10",
- "2010-01-01T01:35:00Z\t1\t20",
- "2010-01-01T02:30:00Z\t1\t30",
-
- "2010-01-01T01:00:00Z\t2\t10",
- "2010-01-01T01:31:00Z\t2\t20",
- "2010-01-01T02:10:00Z\t2\t30",
- "2010-01-01T02:40:30Z\t2\t40",
- "2010-01-01T03:30:00Z\t2\t50",
-
- "2010-01-01T01:00:00Z\t3\t10",
- "2010-01-01T01:01:00Z\t3\t20",
- "2010-01-01T01:02:00Z\t3\t5",
- "2010-01-01T01:10:00Z\t3\t25",
- "2010-01-01T01:15:00Z\t3\t50",
- "2010-01-01T01:25:00Z\t3\t30",
- "2010-01-01T01:30:00Z\t3\t15"
- };
-
- @Test
- public void sessionizeTest() throws Exception
- {
- PigTest test = createPigTestFromString(sessionizeTest,
- "TIME_WINDOW=30m",
- "JAR_PATH=" + getJarPath(),
- "TIME_TYPE=chararray");
-
- this.writeLinesToFile("input",
- inputData);
-
- test.runScript();
-
- HashMap<Integer,HashMap<Integer,Boolean>> userValues = new HashMap<Integer,HashMap<Integer,Boolean>>();
-
- for (Tuple t : this.getLinesForAlias(test, "max_value"))
- {
- Integer userId = (Integer)t.get(0);
- Integer max = (Integer)t.get(1);
- if (!userValues.containsKey(userId))
- {
- userValues.put(userId, new HashMap<Integer,Boolean>());
- }
- userValues.get(userId).put(max, true);
- }
-
- assertEquals(userValues.get(1).size(), 2);
- assertEquals(userValues.get(2).size(), 5);
- assertEquals(userValues.get(3).size(), 1);
-
- assertTrue(userValues.get(1).containsKey(20));
- assertTrue(userValues.get(1).containsKey(30));
-
- assertTrue(userValues.get(2).containsKey(10));
- assertTrue(userValues.get(2).containsKey(20));
- assertTrue(userValues.get(2).containsKey(30));
- assertTrue(userValues.get(2).containsKey(40));
- assertTrue(userValues.get(2).containsKey(50));
-
- assertTrue(userValues.get(3).containsKey(50));
- }
-
- private SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
-
- @Test
- public void sessionizeLongTest() throws Exception
- {
- PigTest test = createPigTestFromString(sessionizeTest,
- "TIME_WINDOW=30m",
- "JAR_PATH=" + getJarPath(),
- "TIME_TYPE=long");
-
- List<String> lines = new ArrayList<String>();
-
- for (String line : inputData)
- {
- String[] parts = line.split("\t");
- Assert.assertEquals(3, parts.length);
- parts[0] = Long.toString(dateFormat.parse(parts[0]).getTime());
- lines.add(StringUtils.join(parts,"\t"));
- }
-
- this.writeLinesToFile("input",
- lines.toArray(new String[]{}));
-
- test.runScript();
-
- HashMap<Integer,HashMap<Integer,Boolean>> userValues = new HashMap<Integer,HashMap<Integer,Boolean>>();
-
- for (Tuple t : this.getLinesForAlias(test, "max_value"))
- {
- Integer userId = (Integer)t.get(0);
- Integer max = (Integer)t.get(1);
- if (!userValues.containsKey(userId))
- {
- userValues.put(userId, new HashMap<Integer,Boolean>());
- }
- userValues.get(userId).put(max, true);
- }
-
- assertEquals(userValues.get(1).size(), 2);
- assertEquals(userValues.get(2).size(), 5);
-
- assertTrue(userValues.get(1).containsKey(20));
- assertTrue(userValues.get(1).containsKey(30));
-
- assertTrue(userValues.get(2).containsKey(10));
- assertTrue(userValues.get(2).containsKey(20));
- assertTrue(userValues.get(2).containsKey(30));
- assertTrue(userValues.get(2).containsKey(40));
- assertTrue(userValues.get(2).containsKey(50));
- }
-
- @Test
- public void sessionizeExecTest() throws Exception
- {
- Sessionize sessionize = new Sessionize("30m");
- Tuple input = TupleFactory.getInstance().newTuple(1);
- DataBag inputBag = BagFactory.getInstance().newDefaultBag();
- input.set(0,inputBag);
-
- Tuple item;
- List<Tuple> result;
- DateTime dt;
-
- // test same session id
- inputBag.clear();
- dt = new DateTime();
- item = TupleFactory.getInstance().newTuple(1);
- item.set(0, dt.getMillis());
- inputBag.add(item);
- item = TupleFactory.getInstance().newTuple(1);
- item.set(0, dt.plusMinutes(28).getMillis());
- inputBag.add(item);
- result = toList(sessionize.exec(input));
-
- Assert.assertEquals(2, result.size());
- Assert.assertEquals(2,result.get(0).size());
- Assert.assertEquals(2,result.get(1).size());
- // session ids match
- Assert.assertTrue(result.get(0).get(1).equals(result.get(1).get(1)));
-
- // test different session id
- inputBag.clear();
- dt = new DateTime();
- item = TupleFactory.getInstance().newTuple(1);
- item.set(0, dt.getMillis());
- inputBag.add(item);
- item = TupleFactory.getInstance().newTuple(1);
- item.set(0, dt.plusMinutes(31).getMillis());
- inputBag.add(item);
- result = toList(sessionize.exec(input));
-
- Assert.assertEquals(2, result.size());
- Assert.assertEquals(2,result.get(0).size());
- Assert.assertEquals(2,result.get(1).size());
- // session ids don't match
- Assert.assertFalse(result.get(0).get(1).equals(result.get(1).get(1)));
- }
-
- @Test
- public void sessionizeAccumulateTest() throws Exception
- {
- Sessionize sessionize = new Sessionize("30m");
- Tuple input = TupleFactory.getInstance().newTuple(1);
- DataBag inputBag = BagFactory.getInstance().newDefaultBag();
- input.set(0,inputBag);
-
- Tuple item;
- List<Tuple> result;
- DateTime dt;
-
- // test same session id
- dt = new DateTime();
- item = TupleFactory.getInstance().newTuple(1);
- item.set(0, dt.getMillis());
- inputBag.add(item);
- sessionize.accumulate(input);
- inputBag.clear();
- item = TupleFactory.getInstance().newTuple(1);
- item.set(0, dt.plusMinutes(28).getMillis());
- inputBag.add(item);
- sessionize.accumulate(input);
- inputBag.clear();
- result = toList(sessionize.getValue());
-
- Assert.assertEquals(2, result.size());
- Assert.assertEquals(2,result.get(0).size());
- Assert.assertEquals(2,result.get(1).size());
- // session ids match
- Assert.assertTrue(result.get(0).get(1).equals(result.get(1).get(1)));
-
- // test different session id
- sessionize.cleanup();
- dt = new DateTime();
- item = TupleFactory.getInstance().newTuple(1);
- item.set(0, dt.getMillis());
- inputBag.add(item);
- sessionize.accumulate(input);
- inputBag.clear();
- item = TupleFactory.getInstance().newTuple(1);
- item.set(0, dt.plusMinutes(31).getMillis());
- inputBag.add(item);
- sessionize.accumulate(input);
- inputBag.clear();
- result = toList(sessionize.getValue());
-
- Assert.assertEquals(2, result.size());
- Assert.assertEquals(2,result.get(0).size());
- Assert.assertEquals(2,result.get(1).size());
- // session ids don't match
- Assert.assertFalse(result.get(0).get(1).equals(result.get(1).get(1)));
-
- sessionize.cleanup();
- Assert.assertEquals(0,sessionize.getValue().size());
- }
-
- private List<Tuple> toList(DataBag bag)
- {
- List<Tuple> result = new ArrayList<Tuple>();
- for (Tuple t : bag)
- {
- result.add(t);
- }
- return result;
- }
-
- /**
- register $JAR_PATH
-
- define SessionCount datafu.pig.sessions.SessionCount('$TIME_WINDOW');
-
- views = LOAD 'input' AS (user_id:int, page_id:int, time:chararray);
-
- views_grouped = GROUP views BY (user_id, page_id);
- view_counts = foreach views_grouped {
- views = order views by time;
- generate group.user_id as user_id, group.page_id as page_id, SessionCount(views.(time)) as count;
- }
-
- STORE view_counts INTO 'output';
- */
- @Multiline
- private String sessionCountPageViewsTest;
-
- @Test
- public void sessionCountPageViewsTest() throws Exception
- {
- PigTest test = createPigTestFromString(sessionCountPageViewsTest,
- "TIME_WINDOW=30m",
- "JAR_PATH=" + getJarPath());
-
- String[] input = {
- "1\t100\t2010-01-01T01:00:00Z",
- "1\t100\t2010-01-01T01:15:00Z",
- "1\t100\t2010-01-01T01:31:00Z",
- "1\t100\t2010-01-01T01:35:00Z",
- "1\t100\t2010-01-01T02:30:00Z",
-
- "1\t101\t2010-01-01T01:00:00Z",
- "1\t101\t2010-01-01T01:31:00Z",
- "1\t101\t2010-01-01T02:10:00Z",
- "1\t101\t2010-01-01T02:40:30Z",
- "1\t101\t2010-01-01T03:30:00Z",
-
- "1\t102\t2010-01-01T01:00:00Z",
- "1\t102\t2010-01-01T01:01:00Z",
- "1\t102\t2010-01-01T01:02:00Z",
- "1\t102\t2010-01-01T01:10:00Z",
- "1\t102\t2010-01-01T01:15:00Z",
- "1\t102\t2010-01-01T01:25:00Z",
- "1\t102\t2010-01-01T01:30:00Z"
- };
-
- String[] output = {
- "(1,100,2)",
- "(1,101,5)",
- "(1,102,1)"
- };
-
- test.assertOutput("views",input,"view_counts",output);
- }
-
- @Test
- public void sessionCountExecTest() throws Exception
- {
- SessionCount sessionize = new SessionCount("30m");
- Tuple input = TupleFactory.getInstance().newTuple(1);
- DataBag inputBag = BagFactory.getInstance().newDefaultBag();
- input.set(0,inputBag);
-
- Tuple item;
- DateTime dt;
-
- // test same session id
- inputBag.clear();
- dt = new DateTime();
- item = TupleFactory.getInstance().newTuple(1);
- item.set(0, dt.getMillis());
- inputBag.add(item);
- item = TupleFactory.getInstance().newTuple(1);
- item.set(0, dt.plusMinutes(28).getMillis());
- inputBag.add(item);
- Assert.assertEquals(1L,sessionize.exec(input).longValue());
-
- // test different session id
- inputBag.clear();
- dt = new DateTime();
- item = TupleFactory.getInstance().newTuple(1);
- item.set(0, dt.getMillis());
- inputBag.add(item);
- item = TupleFactory.getInstance().newTuple(1);
- item.set(0, dt.plusMinutes(31).getMillis());
- inputBag.add(item);
- Assert.assertEquals(2L,sessionize.exec(input).longValue());
- }
-
- @Test
- public void sessionCountAccumulateTest() throws Exception
- {
- SessionCount sessionize = new SessionCount("30m");
- Tuple input = TupleFactory.getInstance().newTuple(1);
- DataBag inputBag = BagFactory.getInstance().newDefaultBag();
- input.set(0,inputBag);
-
- Tuple item;
- DateTime dt;
-
- // test same session id
- dt = new DateTime();
- item = TupleFactory.getInstance().newTuple(1);
- item.set(0, dt.getMillis());
- inputBag.add(item);
- sessionize.accumulate(input);
- inputBag.clear();
- item = TupleFactory.getInstance().newTuple(1);
- item.set(0, dt.plusMinutes(28).getMillis());
- inputBag.add(item);
- sessionize.accumulate(input);
- inputBag.clear();
- Assert.assertEquals(1L,sessionize.getValue().longValue());
-
- // test different session id
- sessionize.cleanup();
- dt = new DateTime();
- item = TupleFactory.getInstance().newTuple(1);
- item.set(0, dt.getMillis());
- inputBag.add(item);
- sessionize.accumulate(input);
- inputBag.clear();
- item = TupleFactory.getInstance().newTuple(1);
- item.set(0, dt.plusMinutes(31).getMillis());
- inputBag.add(item);
- sessionize.accumulate(input);
- inputBag.clear();
- Assert.assertEquals(2L,sessionize.exec(input).longValue());
-
- sessionize.cleanup();
- Assert.assertEquals(0,sessionize.getValue().longValue());
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/sets/SetTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/sets/SetTests.java b/test/pig/datafu/test/pig/sets/SetTests.java
deleted file mode 100644
index de6aa51..0000000
--- a/test/pig/datafu/test/pig/sets/SetTests.java
+++ /dev/null
@@ -1,330 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.sets;
-
-import java.util.Arrays;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import datafu.pig.sets.SetIntersect;
-import datafu.test.pig.PigTests;
-
-public class SetTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- define SetIntersect datafu.pig.sets.SetIntersect();
-
- data = LOAD 'input' AS (B1:bag{T:tuple(val1:int,val2:int)},B2:bag{T:tuple(val1:int,val2:int)});
-
- data2 = FOREACH data GENERATE SetIntersect(B1,B2);
-
- STORE data2 INTO 'output';
- */
- @Multiline
- private String setIntersectTest;
-
- @Test
- public void setIntersectTest() throws Exception
- {
- PigTest test = createPigTestFromString(setIntersectTest);
-
- writeLinesToFile("input",
- "{(1,10),(2,20),(3,30),(4,40),(5,50),(6,60)}\t{(0,0),(2,20),(4,40),(8,80)}",
- "{(1,10),(1,10),(2,20),(3,30),(3,30),(4,40),(4,40)}\t{(1,10),(3,30)}");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "({(2,20),(4,40)})",
- "({(1,10),(3,30)})");
- }
-
- /**
- register $JAR_PATH
-
- define SetIntersect datafu.pig.sets.SetIntersect();
-
- docs = LOAD 'docs' USING PigStorage(',') AS (id:int, line:chararray);
- B = FOREACH docs GENERATE id, line;
- C = FOREACH B GENERATE id, TOKENIZE(line) as gu;
-
- filtered = FOREACH C {
- uniq = DISTINCT gu;
- GENERATE id, uniq;
- }
-
- query = LOAD 'query' AS (line_query:chararray);
- bag_query = FOREACH query GENERATE TOKENIZE(line_query) AS query;
- -- sort the bag of tokens, since SetIntersect requires it
- bag_query = FOREACH bag_query {
- query_sorted = ORDER query BY token;
- GENERATE query_sorted;
- }
-
- result = FOREACH filtered {
- -- sort the tokens, since SetIntersect requires it
- tokens_sorted = ORDER uniq BY token;
- GENERATE id,
- SIZE(SetIntersect(tokens_sorted,bag_query.query_sorted)) as cnt;
- }
-
- DUMP result;
-
- */
- @Multiline
- private String setIntersectTestExample;
-
- @Test
- public void setIntersectTestExample() throws Exception
- {
- PigTest test = createPigTestFromString(setIntersectTestExample);
-
- // document with words to check for
- writeLinesToFile("docs",
- "1,word1 word4 word2 word1",
- "2,word2 word6 word1 word5 word3 word7",
- "3,word1 word3 word4 word5");
-
- // query to apply to document
- writeLinesToFile("query",
- "word2 word7 word5");
-
- test.runScript();
-
- assertOutput(test, "result",
- "(1,1)",
- "(2,3)",
- "(3,1)");
- }
-
- @Test
- public void setIntersectEmptyBagsTest() throws Exception
- {
- PigTest test = createPigTestFromString(setIntersectTest);
-
- writeLinesToFile("input",
- "{(1,10),(2,20),(3,30),(4,40),(5,50),(6,60)}\t{(0,0),(2,20),(4,40),(8,80)}",
- "{(1,10),(1,10),(2,20),(3,30),(3,30),(4,40),(4,40)}\t{(100,10),(300,30)}",
- "{(1,10),(2,20)}\t{(1,10),(2,20)}",
- "{(1,10),(2,20)}\t{}",
- "{}\t{(1,10),(2,20)}",
- "{}\t{}");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "({(2,20),(4,40)})",
- "({})",
- "({(1,10),(2,20)})",
- "({})",
- "({})",
- "({})");
- }
-
- @Test
- public void testIntersectWithNullTuples() throws Exception {
- DataBag one = BagFactory.getInstance().newDefaultBag();
- DataBag two = BagFactory.getInstance().newDefaultBag();
-
- Tuple input = TupleFactory.getInstance().newTuple(Arrays.asList(one, two));
- DataBag output = new SetIntersect().exec(input);
- Assert.assertEquals(0, output.size());
- }
-
- @Test(expectedExceptions=org.apache.pig.impl.logicalLayer.FrontendException.class)
- public void setIntersectOutOfOrderTest() throws Exception
- {
- PigTest test = createPigTestFromString(setIntersectTest);
-
- this.writeLinesToFile("input",
- "{(1,10),(3,30),(2,20),(4,40),(5,50),(6,60)}\t{(0,0),(2,20),(4,40),(8,80)}");
-
- test.runScript();
-
- this.getLinesForAlias(test, "data2");
- }
-
- /**
- register $JAR_PATH
-
- define SetUnion datafu.pig.sets.SetUnion();
-
- data = LOAD 'input' AS (B1:bag{T:tuple(val1:int,val2:int)},B2:bag{T:tuple(val1:int,val2:int)});
-
- --dump data
-
- data2 = FOREACH data GENERATE SetUnion(B1,B2) AS C;
- data2 = FOREACH data2 {
- C = ORDER C BY val1 ASC, val2 ASC;
- generate C;
- }
-
- --dump data2
-
- STORE data2 INTO 'output';
- */
- @Multiline
- private String setUnionTest;
-
- @Test
- public void setUnionTest() throws Exception
- {
- PigTest test = createPigTestFromString(setUnionTest);
-
- writeLinesToFile("input",
- "{(1,10),(1,20),(1,30),(1,40),(1,50),(1,60),(1,80)}\t{(1,1),(1,20),(1,25),(1,25),(1,25),(1,40),(1,70),(1,80)}");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "({(1,1),(1,10),(1,20),(1,25),(1,30),(1,40),(1,50),(1,60),(1,70),(1,80)})");
- }
-
- @Test
- public void setUnionEmptyBagsTest() throws Exception
- {
- PigTest test = createPigTestFromString(setUnionTest);
-
- writeLinesToFile("input",
- "{(1,10),(1,20),(1,30),(1,40),(1,50),(1,60),(1,80)}\t{}",
- "{}\t{(1,10),(1,20),(1,30),(1,40),(1,50)}",
- "{}\t{}");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "({(1,10),(1,20),(1,30),(1,40),(1,50),(1,60),(1,80)})",
- "({(1,10),(1,20),(1,30),(1,40),(1,50)})",
- "({})");
- }
-
- /**
- register $JAR_PATH
-
- define SetDifference datafu.pig.sets.SetDifference();
-
- data = LOAD 'input' AS (B1:bag{T:tuple(val:int)},B2:bag{T:tuple(val:int)});
-
- data2 = FOREACH data GENERATE SetDifference(B1,B2);
-
- STORE data2 INTO 'output';
- */
- @Multiline
- private String setDifferenceTwoBagsTest;
-
- @Test
- public void setDifferenceTwoBagsTest() throws Exception
- {
- PigTest test = createPigTestFromString(setDifferenceTwoBagsTest);
-
- writeLinesToFile("input",
- "{(1),(2),(3)}\t",
- "{(1),(2),(3)}\t{}",
- "\t{(1),(2),(3)}",
- "{}\t{(1),(2),(3)}",
- "{(1),(2),(3)}\t{(1)}",
- "{(1),(2),(3)}\t{(1),(2)}",
- "{(1),(2),(3)}\t{(1),(2),(3)}",
- "{(1),(2),(3)}\t{(1),(2),(3),(4)}",
- "{(1),(2),(3),(4),(5),(6)}\t{(3),(4)}");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "({(1),(2),(3)})",
- "({(1),(2),(3)})",
- "({})",
- "({})",
- "({(2),(3)})",
- "({(3)})",
- "({})",
- "({})",
- "({(1),(2),(5),(6)})");
- }
-
- /**
- register $JAR_PATH
-
- define SetDifference datafu.pig.sets.SetDifference();
-
- data = LOAD 'input' AS (B1:bag{T:tuple(val:int)},B2:bag{T:tuple(val:int)},B3:bag{T:tuple(val:int)});
-
- data2 = FOREACH data GENERATE SetDifference(B1,B2,B3);
-
- STORE data2 INTO 'output';
- */
- @Multiline
- private String setDifferenceThreeBagsTest;
-
- @Test
- public void setDifferenceThreeBagsTest() throws Exception
- {
- PigTest test = createPigTestFromString(setDifferenceThreeBagsTest);
-
- writeLinesToFile("input",
- "{(1),(2),(3)}\t\t",
- "{(1),(2),(3)}\t{}\t",
- "{(1),(2),(3)}\t\t{}",
- "{(1),(2),(3)}\t{}\t{}",
- "{(1),(2),(2),(2),(3),(3)}\t{}\t{}",
-
- "{(1),(2),(3)}\t{(2)}\t{}",
- "{(1),(2),(3)}\t{}\t{(2)}",
-
- "{(1),(2),(3)}\t{(2)}\t{(3)}",
-
- "{(1),(2),(3)}\t{(2),(2)}\t{(3),(3)}",
- "{(1),(1),(1),(2),(2),(3),(3),(3)}\t{(2),(2)}\t{(3),(3)}",
- "{(1),(2),(3)}\t{(0),(2)}\t{(3)}",
-
- "{(1),(2),(3)}\t{(0),(2)}\t{(1),(3)}",
- "{(1),(2),(3)}\t{(0),(2)}\t{(1),(3),(4)}");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "({(1),(2),(3)})",
- "({(1),(2),(3)})",
- "({(1),(2),(3)})",
- "({(1),(2),(3)})",
- "({(1),(2),(3)})",
-
- "({(1),(3)})",
- "({(1),(3)})",
-
- "({(1)})",
-
- "({(1)})",
- "({(1)})",
- "({(1)})",
-
- "({})",
- "({})");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/stats/EstimationTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/stats/EstimationTests.java b/test/pig/datafu/test/pig/stats/EstimationTests.java
deleted file mode 100644
index 258f320..0000000
--- a/test/pig/datafu/test/pig/stats/EstimationTests.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.stats;
-
-import java.util.List;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.test.pig.PigTests;
-import static org.testng.Assert.*;
-
-public class EstimationTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- define HyperLogLogPlusPlus datafu.pig.stats.HyperLogLogPlusPlus();
-
- data_in = LOAD 'input' as (val:int);
-
- data_out = FOREACH (GROUP data_in ALL) GENERATE
- HyperLogLogPlusPlus(data_in) as cardinality;
-
- data_out = FOREACH data_out GENERATE cardinality;
-
- STORE data_out into 'output';
- */
- @Multiline private String hyperLogLogTest;
-
- @Test
- public void hyperLogLogTest() throws Exception
- {
- PigTest test = createPigTestFromString(hyperLogLogTest);
-
- int count = 1000000;
- String[] input = new String[count];
- for (int i=0; i<count; i++)
- {
- input[i] = Integer.toString(i*10);
- }
-
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- double error = Math.abs(count-((Long)output.get(0).get(0)))/(double)count;
- System.out.println("error: " + error*100.0 + "%");
- assertTrue(error < 0.01);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/stats/MarkovPairTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/stats/MarkovPairTests.java b/test/pig/datafu/test/pig/stats/MarkovPairTests.java
deleted file mode 100644
index b2f618e..0000000
--- a/test/pig/datafu/test/pig/stats/MarkovPairTests.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.stats;
-
-import static org.testng.Assert.*;
-
-import java.util.Iterator;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.test.pig.PigTests;
-
-public class MarkovPairTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- define markovPairs datafu.pig.stats.MarkovPairs();
-
- data = load 'input' as $schema;
- --describe data;
-
- data_out1 = foreach data generate data as orig_bag;
- --describe data_out1;
-
- data_out = foreach data_out1 generate markovPairs(orig_bag) as markov_bag;
- --describe data_out;
-
- store data_out into 'output';
- */
- @Multiline private String markovPairDefault;
-
- @Test
- public void markovPairDefaultTest() throws Exception
- {
- PigTest test = createPigTestFromString(markovPairDefault,
- "schema=(data: bag {t: tuple(val:int)})");
-
- writeLinesToFile("input", "{(10),(20),(30),(40),(50),(60)}");
-
- String[] expectedOutput = {
- "({((10),(20)),((20),(30)),((30),(40)),((40),(50)),((50),(60))})"
- };
-
- test.runScript();
-
- Iterator<Tuple> actualOutput = test.getAlias("data_out");
-
- assertTuplesMatch(expectedOutput, actualOutput);
- }
-
- @Test
- public void markovPairMultipleInput() throws Exception
- {
- PigTest test = createPigTestFromString(markovPairDefault,
- "schema=(data: bag {t: tuple(val1:int,val2:int)})");
-
- writeLinesToFile("input", "{(10,100),(20,200),(30,300),(40,400),(50,500),(60,600)}");
-
- String[] expectedOutput = {
- "({((10,100),(20,200)),((20,200),(30,300)),((30,300),(40,400)),((40,400),(50,500)),((50,500),(60,600))})"
- };
-
-
- test.runScript();
-
- Iterator<Tuple> actualOutput = test.getAlias("data_out");
-
- assertTuplesMatch(expectedOutput, actualOutput);
- }
-
- /**
- register $JAR_PATH
-
- define markovPairs datafu.pig.stats.MarkovPairs('$lookahead');
-
- data = load 'input' as $schema;
- --describe data;
-
- data_out1 = foreach data generate data as orig_bag;
- --describe data_out1;
-
- data_out = foreach data_out1 generate markovPairs(orig_bag) as markov_bag;
- --describe data_out;
-
- store data_out into 'output';
- */
- @Multiline private String markovPairLookahead;
-
- @Test
- public void markovPairLookaheadTest() throws Exception
- {
- PigTest test = createPigTestFromString(markovPairLookahead,
- "schema=(data: bag {t: tuple(val:int)})",
- "lookahead=3");
-
- writeLinesToFile("input", "{(10),(20),(30),(40),(50)}");
-
- String[] expectedOutput = {
- "({((10),(20)),((10),(30)),((10),(40)),((20),(30)),((20),(40)),((20),(50)),((30),(40)),((30),(50)),((40),(50))})"
- };
-
- test.runScript();
-
- Iterator<Tuple> actualOutput = test.getAlias("data_out");
-
- assertTuplesMatch(expectedOutput, actualOutput);
- }
-
- private void assertTuplesMatch(String[] expectedOutput, Iterator<Tuple> actualOutput)
- {
- Iterator<Tuple> tuples = actualOutput;
-
- for (String outputLine : expectedOutput)
- {
- assertTrue(tuples.hasNext());
- Tuple outputTuple = tuples.next();
- System.out.println(String.format("expected: %s", outputLine));
- System.out.println(String.format("actual: %s", outputTuple.toString()));
- assertEquals(outputLine,outputTuple.toString());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/stats/QuantileTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/stats/QuantileTests.java b/test/pig/datafu/test/pig/stats/QuantileTests.java
deleted file mode 100644
index 1814016..0000000
--- a/test/pig/datafu/test/pig/stats/QuantileTests.java
+++ /dev/null
@@ -1,696 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.stats;
-
-import static org.testng.Assert.*;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import junit.framework.Assert;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.pig.stats.QuantileUtil;
-import datafu.pig.stats.StreamingQuantile;
-import datafu.test.pig.PigTests;
-
-public class QuantileTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- define Quantile datafu.pig.stats.Quantile($QUANTILES);
-
- data_in = LOAD 'input' as (val:int);
-
- --describe data_in;
-
- data_out = GROUP data_in ALL;
-
- --describe data_out;
-
- data_out = FOREACH data_out {
- sorted = ORDER data_in BY val;
- GENERATE Quantile(sorted) as quantiles;
- }
- data_out = FOREACH data_out GENERATE FLATTEN(quantiles);
-
- --describe data_out;
-
- STORE data_out into 'output';
- */
- @Multiline private String quantileTest;
-
- @Test
- public void quantileTest() throws Exception
- {
- PigTest test = createPigTestFromString(quantileTest,
- "QUANTILES='0.0','0.25','0.5','0.75','1.0'");
-
- String[] input = {"1","2","3","4","10","5","6","7","8","9"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(1.0,3.0,5.5,8.0,10.0)");
- }
-
- @Test
- public void quantile2Test() throws Exception
- {
- PigTest test = createPigTestFromString(quantileTest,
- "QUANTILES='5'");
-
- String[] input = {"1","2","3","4","10","5","6","7","8","9"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(1.0,3.0,5.5,8.0,10.0)");
- }
-
- @Test
- public void quantile3Test() throws Exception {
- PigTest test = createPigTestFromString(quantileTest,
- "QUANTILES='0.0013','0.0228','0.1587','0.5','0.8413','0.9772','0.9987'");
-
- List<String> input = new ArrayList<String>();
- for (int i=100000; i>=0; i--)
- {
- input.add(Integer.toString(i));
- }
-
- writeLinesToFile("input", input.toArray(new String[0]));
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(130.0,2280.0,15870.0,50000.0,84130.0,97720.0,99870.0)");
- }
-
- @Test
- public void quantile4aTest() throws Exception
- {
- PigTest test = createPigTestFromString(quantileTest,
- "QUANTILES='4'");
-
- String[] input = {"1","2","3","4","10","5","6","7","8","9"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(1.0,4.0,7.0,10.0)");
- }
-
- @Test
- public void quantile4bTest() throws Exception
- {
- PigTest test = createPigTestFromString(quantileTest,
- "QUANTILES='0.0','0.333','0.666','1.0'");
-
- String[] input = {"1","2","3","4","10","5","6","7","8","9"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(1.0,4.0,7.0,10.0)");
- }
-
- @Test
- public void quantile5aTest() throws Exception
- {
- PigTest test = createPigTestFromString(quantileTest,
- "QUANTILES='10'");
-
- String[] input = {"1","2","3","4","10","5","6","7","8","9"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0)");
- }
-
- @Test
- public void quantile5bTest() throws Exception
- {
- PigTest test = createPigTestFromString(quantileTest,
- "QUANTILES='0.0','0.111','0.222','0.333','0.444','0.555','0.666','0.777','0.888','1.0'");
-
- String[] input = {"1","2","3","4","10","5","6","7","8","9"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0)");
- }
-
- /**
- register $JAR_PATH
-
- define Median datafu.pig.stats.Median();
-
- data_in = LOAD 'input' as (val:int);
-
- --describe data_in;
-
- data_out = GROUP data_in ALL;
-
- --describe data_out;
-
- data_out = FOREACH data_out {
- sorted = ORDER data_in BY val;
- GENERATE Median(sorted) as medians;
- }
- data_out = FOREACH data_out GENERATE FLATTEN(medians);
-
- --describe data_out;
-
- STORE data_out into 'output';
- */
- @Multiline private String medianTest;
-
- @Test
- public void medianTest() throws Exception
- {
- PigTest test = createPigTestFromString(medianTest);
-
- String[] input = {"4","5","6","9","10","7","8","2","3","1"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(5.5)");
- }
-
- /**
- register $JAR_PATH
-
- define Median datafu.pig.stats.StreamingMedian();
-
- data_in = LOAD 'input' as (val:int);
-
- --describe data_in;
-
- data_out = GROUP data_in ALL;
-
- --describe data_out;
-
- data_out = FOREACH data_out {
- GENERATE Median(data_in) as medians;
- }
- data_out = FOREACH data_out GENERATE FLATTEN(medians);
-
- --describe data_out;
-
- STORE data_out into 'output';
- */
- @Multiline private String streamingMedianTest;
-
- @Test
- public void streamingMedianTest() throws Exception
- {
- PigTest test = createPigTestFromString(streamingMedianTest);
-
- String[] input = {"0","4","5","6","9","10","7","8","2","3","1"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(5.0)");
- }
-
- /**
- register $JAR_PATH
-
- define Quantile datafu.pig.stats.StreamingQuantile($QUANTILES);
-
- data_in = LOAD 'input' as (val:int);
-
- --describe data_in;
-
- data_out = GROUP data_in ALL;
-
- --describe data_out;
-
- data_out = FOREACH data_out GENERATE Quantile(data_in.val) as quantiles;
- data_out = FOREACH data_out GENERATE FLATTEN(quantiles);
-
- --describe data_out;
-
- STORE data_out into 'output';
- */
- @Multiline private String streamingQuantileTest;
-
- @Test
- public void streamingQuantileTest() throws Exception {
- PigTest test = createPigTestFromString(streamingQuantileTest,
- "QUANTILES='5'");
-
- String[] input = {"1","2","3","4","10","5","6","7","8","9"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(1.0,3.0,5.0,8.0,10.0)");
- }
-
- @Test
- public void streamingQuantile2Test() throws Exception {
- PigTest test = createPigTestFromString(streamingQuantileTest,
- "QUANTILES='0.5','0.75','1.0'");
-
- String[] input = {"1","2","3","4","10","5","6","7","8","9"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(5.0,8.0,10.0)");
- }
-
- @Test
- public void streamingQuantile3Test() throws Exception {
- PigTest test = createPigTestFromString(streamingQuantileTest,
- "QUANTILES='0.07','0.03','0.37','1.0','0.0'");
-
- List<String> input = new ArrayList<String>();
- for (int i=1000; i>=1; i--)
- {
- input.add(Integer.toString(i));
- }
-
- writeLinesToFile("input", input.toArray(new String[0]));
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(70.0,30.0,370.0,1000.0,1.0)");
- }
-
- @Test
- public void streamingQuantile4Test() throws Exception {
- PigTest test = createPigTestFromString(streamingQuantileTest,
- "QUANTILES='0.0013','0.0228','0.1587','0.5','0.8413','0.9772','0.9987'");
-
- List<String> input = new ArrayList<String>();
- for (int i=100000; i>=0; i--)
- {
- input.add(Integer.toString(i));
- }
-
- writeLinesToFile("input", input.toArray(new String[0]));
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(130.0,2280.0,15870.0,50000.0,84130.0,97720.0,99870.0)");
- }
-
- @Test
- public void streamingQuantile5aTest() throws Exception {
- PigTest test = createPigTestFromString(streamingQuantileTest,
- "QUANTILES='10'");
-
- String[] input = {"1","2","3","4","10","5","6","7","8","9"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0)");
- }
-
- @Test
- public void streamingQuantile5bTest() throws Exception {
- PigTest test = createPigTestFromString(streamingQuantileTest,
- "QUANTILES='0.0','0.111','0.222','0.333','0.444','0.555','0.666','0.777','0.888','1.0'");
-
- String[] input = {"1","2","3","4","10","5","6","7","8","9"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0)");
- }
-
- @Test
- public void streamingQuantile6Test() throws Exception {
- PigTest test = createPigTestFromString(streamingQuantileTest,
- "QUANTILES='0.0','0.333','0.666','1.0'");
-
- String[] input = {"1","2","3","4","10","5","6","7","8","9"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(1.0,4.0,7.0,10.0)");
- }
-
- @Test
- public void streamingQuantile7Test() throws Exception {
- PigTest test = createPigTestFromString(streamingQuantileTest,
- "QUANTILES='4'");
-
- String[] input = {"1","2","3","4","10","5","6","7","8","9"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(1.0,4.0,7.0,10.0)");
- }
-
- @Test
- public void streamingQuantileExecTest() throws Exception {
- StreamingQuantile quantile = new StreamingQuantile("4");
-
- DataBag bag;
- Tuple input;
- Tuple result;
-
- bag = BagFactory.getInstance().newDefaultBag();
- for (int i=1; i<=10; i++)
- {
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, i);
- bag.add(t);
- }
- input = TupleFactory.getInstance().newTuple(bag);
- result = quantile.exec(input);
- Assert.assertEquals(4, result.size());
- Assert.assertEquals(1.0, result.get(0));
- Assert.assertEquals(4.0, result.get(1));
- Assert.assertEquals(7.0, result.get(2));
- Assert.assertEquals(10.0, result.get(3));
-
- // do twice to check cleanup works
-
- bag = BagFactory.getInstance().newDefaultBag();
- for (int i=11; i<=20; i++)
- {
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, i);
- bag.add(t);
- }
- input = TupleFactory.getInstance().newTuple(bag);
- result = quantile.exec(input);
- Assert.assertEquals(4, result.size());
- Assert.assertEquals(11.0, result.get(0));
- Assert.assertEquals(14.0, result.get(1));
- Assert.assertEquals(17.0, result.get(2));
- Assert.assertEquals(20.0, result.get(3));
- }
-
- @Test
- public void streamingQuantileAccumulateTest() throws Exception {
- StreamingQuantile quantile = new StreamingQuantile("4");
-
- Tuple result;
-
- for (int i=1; i<=10; i++)
- {
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, i);
- DataBag bag = BagFactory.getInstance().newDefaultBag();
- bag.add(t);
- Tuple input = TupleFactory.getInstance().newTuple(bag);
- quantile.accumulate(input);
- }
- result = quantile.getValue();
- Assert.assertEquals(4, result.size());
- Assert.assertEquals(1.0, result.get(0));
- Assert.assertEquals(4.0, result.get(1));
- Assert.assertEquals(7.0, result.get(2));
- Assert.assertEquals(10.0, result.get(3));
-
- // do twice to check cleanup works
- quantile.cleanup();
-
- for (int i=11; i<=20; i++)
- {
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, i);
- DataBag bag = BagFactory.getInstance().newDefaultBag();
- bag.add(t);
- Tuple input = TupleFactory.getInstance().newTuple(bag);
- quantile.accumulate(input);
- }
- result = quantile.getValue();
- Assert.assertEquals(4, result.size());
- Assert.assertEquals(11.0, result.get(0));
- Assert.assertEquals(14.0, result.get(1));
- Assert.assertEquals(17.0, result.get(2));
- Assert.assertEquals(20.0, result.get(3));
- }
-
- @Test
- public void quantileParamsTest() throws Exception {
- List<Double> quantiles = QuantileUtil.getQuantilesFromParams("5");
-
- assertEquals(quantiles.size(),5);
- assertAboutEqual(quantiles.get(0), 0.0);
- assertAboutEqual(quantiles.get(1), 0.25);
- assertAboutEqual(quantiles.get(2), 0.5);
- assertAboutEqual(quantiles.get(3), 0.75);
- assertAboutEqual(quantiles.get(4), 1.0);
- }
-
- @Test
- public void quantileParamsTest2() throws Exception {
- List<Double> quantiles = QuantileUtil.getQuantilesFromParams("2");
-
- assertEquals(quantiles.size(),2);
- assertAboutEqual(quantiles.get(0), 0.0);
- assertAboutEqual(quantiles.get(1), 1.0);
- }
-
- @Test
- public void quantileParamsTest3() throws Exception {
- List<Double> quantiles = QuantileUtil.getQuantilesFromParams("11");
-
- assertEquals(quantiles.size(),11);
- assertAboutEqual(quantiles.get(0), 0.0);
- assertAboutEqual(quantiles.get(1), 0.1);
- assertAboutEqual(quantiles.get(2), 0.2);
- assertAboutEqual(quantiles.get(3), 0.3);
- assertAboutEqual(quantiles.get(4), 0.4);
- assertAboutEqual(quantiles.get(5), 0.5);
- assertAboutEqual(quantiles.get(6), 0.6);
- assertAboutEqual(quantiles.get(7), 0.7);
- assertAboutEqual(quantiles.get(8), 0.8);
- assertAboutEqual(quantiles.get(9), 0.9);
- assertAboutEqual(quantiles.get(10), 1.0);
- }
-
- @Test
- public void quantileParamsTest4() throws Exception {
- List<Double> quantiles = QuantileUtil.getQuantilesFromParams("10");
-
- assertEquals(quantiles.size(),10);
- assertAboutEqual(quantiles.get(0), 0.0);
- assertAboutEqual(quantiles.get(1), 0.11111);
- assertAboutEqual(quantiles.get(2), 0.22222);
- assertAboutEqual(quantiles.get(3), 0.33333);
- assertAboutEqual(quantiles.get(4), 0.44444);
- assertAboutEqual(quantiles.get(5), 0.55555);
- assertAboutEqual(quantiles.get(6), 0.66666);
- assertAboutEqual(quantiles.get(7), 0.77777);
- assertAboutEqual(quantiles.get(8), 0.88888);
- assertAboutEqual(quantiles.get(9), 1.0);
- }
-
- @Test
- public void quantileParamsTest5() throws Exception {
- List<Double> quantiles = QuantileUtil.getQuantilesFromParams("4");
-
- assertEquals(quantiles.size(),4);
- assertAboutEqual(quantiles.get(0), 0.0);
- assertAboutEqual(quantiles.get(1), 0.333);
- assertAboutEqual(quantiles.get(2), 0.666);
- assertAboutEqual(quantiles.get(3), 1.0);
- }
-
- private void assertAboutEqual(double actual, double expected) {
- assertTrue(Math.abs(actual-expected) < 0.001);
- }
-
- /**
- register $JAR_PATH
-
- define Quantile datafu.pig.stats.$UDF($QUANTILES);
-
- data_in = LOAD 'input' as (val:int);
-
- --describe data_in;
-
- data_out = GROUP data_in ALL;
-
- --describe data_out;
-
- data_out = FOREACH data_out {
- sorted = ORDER data_in BY val;
- GENERATE Quantile(sorted) as quantiles;
- }
- data_out = FOREACH data_out GENERATE FLATTEN(quantiles);
-
- --describe data_out;
-
- data_out = FOREACH data_out GENERATE $EXPECTED_OUTPUT;
-
- STORE data_out into 'output';
- */
- @Multiline private String quantileSchemaTest;
-
- @Test
- public void quantileSchemaTest() throws Exception
- {
- PigTest test = createPigTestFromString(quantileSchemaTest,
- "UDF=Quantile",
- "QUANTILES='0.0','0.5','1.0'",
- "EXPECTED_OUTPUT=quantiles::quantile_0_0, "+
- "quantiles::quantile_0_5, "+
- "quantiles::quantile_1_0");
-
- String[] input = {"1","5","3","4","2"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(1.0,3.0,5.0)");
- }
-
- @Test
- public void quantileSchemaTest2() throws Exception
- {
- PigTest test = createPigTestFromString(quantileSchemaTest,
- "UDF=Quantile",
- "QUANTILES='3'",
- "EXPECTED_OUTPUT=quantiles::quantile_0, "+
- "quantiles::quantile_1, "+
- "quantiles::quantile_2");
-
- String[] input = {"1","5","3","4","2"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(1.0,3.0,5.0)");
- }
-
- @Test
- public void quantileSchemaTest3() throws Exception
- {
- PigTest test = createPigTestFromString(quantileSchemaTest,
- "UDF=StreamingQuantile",
- "QUANTILES='0.0','0.5','1.0'",
- "EXPECTED_OUTPUT=quantiles::quantile_0_0, "+
- "quantiles::quantile_0_5, "+
- "quantiles::quantile_1_0");
-
- String[] input = {"1","5","3","4","2"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(1.0,3.0,5.0)");
- }
-
- @Test
- public void quantileSchemaTest4() throws Exception
- {
- PigTest test = createPigTestFromString(quantileSchemaTest,
- "UDF=StreamingQuantile",
- "QUANTILES='3'",
- "EXPECTED_OUTPUT=quantiles::quantile_0, "+
- "quantiles::quantile_1, "+
- "quantiles::quantile_2");
-
- String[] input = {"1","5","3","4","2"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(1.0,3.0,5.0)");
- }
-}
[14/19] DATAFU-27 Migrate build system to Gradle
Posted by wv...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/text/opennlp/POSTag.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/text/opennlp/POSTag.java b/datafu-pig/src/main/java/datafu/pig/text/opennlp/POSTag.java
new file mode 100644
index 0000000..fb17c63
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/text/opennlp/POSTag.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package datafu.pig.text.opennlp;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import opennlp.tools.postag.POSModel;
+import opennlp.tools.postag.POSTaggerME;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.*;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * The OpenNLP POSTag UDF tags bags of sequential words with parts of speech and confidence levels using the OpenNLP
+ * toolset, and specifically the POSTaggerME class.
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * define TokenizeME datafu.pig.text.opennlp.TokenizeME('data/en-token.bin');
+ * define POSTag datafu.pig.text.opennlp.POSTag('data/en-pos-maxent.bin');
+ *
+ * -- input:
+ * -- (Appetizers during happy hour range from low to high.)
+ * input = LOAD 'input' AS (text:chararray);
+ * --
+ * -- ({(Appetizers),(during),(happy),(hour),(range),(from),(low),(to),(high),(.)})
+ * tokenized = FOREACH input GENERATE TokenizeME(text) AS tokens;
+ * --
+ * -- output:
+ * -- Tuple schema is: (word, tag, confidence)
+ * -- ({(Appetizers,NNP,0.3619277937390988),(during,IN,0.7945543860326094),(happy,JJ,0.9888504792754391),
+ * -- (hour,NN,0.9427455123502427),(range,NN,0.7335527963654751),(from,IN,0.9911576465589752),(low,JJ,0.9652034031895174),
+ * -- (to,IN,0.7005347487371849),(high,JJ,0.8227771746247106),(.,.,0.9900983495480891)})
+ * output = FOREACH tokenized GENERATE POSTag(tokens) AS tagged;
+ * }
+ * </pre>
+ */
+public class POSTag extends EvalFunc<DataBag>
+{
+ private POSTaggerME tagger = null;
+ private static final String MODEL_FILE = "pos";
+ private TupleFactory tf = TupleFactory.getInstance();
+ private BagFactory bf = BagFactory.getInstance();
+ private String modelPath;
+
+ public POSTag(String modelPath) {
+ this.modelPath = modelPath;
+ }
+
+ @Override
+ public List<String> getCacheFiles() {
+ List<String> list = new ArrayList<String>(1);
+ list.add(this.modelPath + "#" + MODEL_FILE);
+ return list;
+ }
+
+ // Enable multiple languages by specifying the model path. See http://text.sourceforge.net/models-1.5/
+ public DataBag exec(Tuple input) throws IOException
+ {
+ DataBag inputBag = null;
+
+ if(input.size() != 1) {
+ throw new IOException();
+ }
+
+ inputBag = (DataBag)input.get(0);
+ DataBag outBag = bf.newDefaultBag();
+ if(this.tagger == null) {
+ String loadFile = CachedFile.getFileName(MODEL_FILE, this.modelPath);
+ InputStream modelIn = new FileInputStream(loadFile);
+ InputStream buffer = new BufferedInputStream(modelIn);
+ POSModel model = new POSModel(buffer);
+ this.tagger = new POSTaggerME(model);
+ }
+
+ // Form an inputString array thing for tagger to act on
+ int bagLength = (int)inputBag.size();
+ String[] words = new String[bagLength];
+
+ Iterator<Tuple> itr = inputBag.iterator();
+ int i = 0;
+ while(itr.hasNext()) {
+ words[i] = (String)itr.next().get(0);
+ i++;
+ }
+
+ // Compute tags and their probabilities
+ String tags[] = this.tagger.tag(words);
+ double probs[] = this.tagger.probs();
+
+ // Build output bag of 3-tuples
+ for(int j = 0; j < tags.length; j++) {
+ Tuple newTuple = tf.newTuple(3);
+ newTuple.set(0, words[j]);
+ newTuple.set(1, tags[j]);
+ newTuple.set(2, probs[j]);
+ outBag.add(newTuple);
+ }
+
+ return outBag;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try
+ {
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.BAG)
+ {
+ throw new RuntimeException("Expected a BAG as input");
+ }
+
+ Schema inputBagSchema = inputFieldSchema.schema;
+
+ if(inputBagSchema == null) {
+ return null;
+ }
+
+ if (inputBagSchema.getField(0).type != DataType.TUPLE)
+ {
+ throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
+ DataType.findTypeName(inputBagSchema.getField(0).type)));
+ }
+
+ Schema inputTupleSchema = inputBagSchema.getField(0).schema;
+
+ if (inputTupleSchema.size() != 1)
+ {
+ throw new RuntimeException("Expected one field for the token data");
+ }
+
+ if (inputTupleSchema.getField(0).type != DataType.CHARARRAY)
+ {
+ throw new RuntimeException(String.format("Expected source to be a CHARARRAY, but instead found %s",
+ DataType.findTypeName(inputTupleSchema.getField(0).type)));
+ }
+
+ Schema tupleSchema = new Schema();
+ tupleSchema.add(new Schema.FieldSchema("token",DataType.CHARARRAY));
+ tupleSchema.add(new Schema.FieldSchema("tag",DataType.CHARARRAY));
+ tupleSchema.add(new Schema.FieldSchema("probability",DataType.DOUBLE));
+
+ return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
+ .getName()
+ .toLowerCase(), input),
+ tupleSchema,
+ DataType.BAG));
+ }
+ catch (FrontendException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/text/opennlp/SentenceDetect.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/text/opennlp/SentenceDetect.java b/datafu-pig/src/main/java/datafu/pig/text/opennlp/SentenceDetect.java
new file mode 100644
index 0000000..50537fd
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/text/opennlp/SentenceDetect.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package datafu.pig.text.opennlp;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.List;
+
+import opennlp.tools.sentdetect.SentenceDetectorME;
+import opennlp.tools.sentdetect.SentenceModel;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.*;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * The OpenNLP SentenceDectectors segment an input paragraph into sentences.
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * define SentenceDetect datafu.pig.text.SentenceDetect('data/en-sent.bin');
+ *
+ * -- input:
+ * -- ("I believe the Masons have infiltrated the Apache PMC. I believe laser beams control cat brains.")
+ * infoo = LOAD 'input' AS (text:chararray);
+
+ * -- output:
+ * -- ({(I believe the Masons have infiltrated the Apache PMC.)(I believe laser beams control cat brains.)})
+ * outfoo = FOREACH input GENERATE SentenceDetect(text) as sentences;
+ * }
+ * </pre>
+ */
+public class SentenceDetect extends EvalFunc<DataBag>
+{
+ private SentenceDetectorME sdetector = null;
+ private static final String MODEL_FILE = "sentences";
+ private TupleFactory tf = TupleFactory.getInstance();
+ private BagFactory bf = BagFactory.getInstance();
+ private String modelPath = null;
+
+ public SentenceDetect(String modelPath) {
+ this.modelPath = modelPath;
+ }
+
+ @Override
+ public List<String> getCacheFiles() {
+ List<String> list = new ArrayList<String>(1);
+ list.add(this.modelPath + "#" + MODEL_FILE);
+ return list;
+ }
+
+ // Enable multiple languages by specifying the model path. See http://text.sourceforge.net/models-1.5/
+ public DataBag exec(Tuple input) throws IOException
+ {
+ if(input.size() != 1) {
+ throw new IOException();
+ }
+
+ String inputString = input.get(0).toString();
+ if(inputString == null || inputString == "") {
+ return null;
+ }
+ DataBag outBag = bf.newDefaultBag();
+ if(sdetector == null) {
+ String loadFile = CachedFile.getFileName(MODEL_FILE, this.modelPath);
+ InputStream is = new FileInputStream(modelPath);
+ InputStream buffer = new BufferedInputStream(is);
+ SentenceModel model = new SentenceModel(buffer);
+ this.sdetector = new SentenceDetectorME(model);
+ }
+ String sentences[] = this.sdetector.sentDetect(inputString);
+ for(String sentence : sentences) {
+ Tuple outTuple = tf.newTuple(sentence);
+ outBag.add(outTuple);
+ }
+ return outBag;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try
+ {
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.CHARARRAY)
+ {
+ throw new RuntimeException("Expected a CHARARRAY as input, but got a " + inputFieldSchema.toString());
+ }
+
+ Schema tupleSchema = new Schema();
+ tupleSchema.add(new Schema.FieldSchema("sentence",DataType.CHARARRAY));
+
+ return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
+ .getName()
+ .toLowerCase(), input),
+ tupleSchema,
+ DataType.BAG));
+ }
+ catch (FrontendException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/text/opennlp/TokenizeME.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/text/opennlp/TokenizeME.java b/datafu-pig/src/main/java/datafu/pig/text/opennlp/TokenizeME.java
new file mode 100644
index 0000000..f1f4257
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/text/opennlp/TokenizeME.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package datafu.pig.text.opennlp;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.List;
+
+import opennlp.tools.tokenize.TokenizerME;
+import opennlp.tools.tokenize.TokenizerModel;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.*;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * The OpenNLP Tokenizers segment an input character sequence into tokens using the OpenNLP TokenizeME class, which is
+ * a probabilistic, 'maximum entropy' classifier.
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * define TokenizeME datafu.pig.text.opennlp.TokenizeME('data/en-token.bin');
+ *
+ * -- input:
+ * -- ("I believe the Masons have infiltrated the Apache PMC.")
+ * infoo = LOAD 'input' AS (text:chararray);
+
+ * -- output:
+ * -- ({(I),(believe),(the),(Masons),(have),(infiltrated),(the),(Apache),(PMC),(.)})
+ * outfoo = FOREACH input GENERATE TokenizeME(text) as tokens;
+ * }
+ * </pre>
+ */
+
+
+
+public class TokenizeME extends EvalFunc<DataBag>
+{
+ private TokenizerME tokenizer = null;
+ private static final String MODEL_FILE = "tokens";
+ private TupleFactory tf = TupleFactory.getInstance();
+ private BagFactory bf = BagFactory.getInstance();
+ private String modelPath;
+
+ public TokenizeME(String modelPath) {
+ this.modelPath = modelPath;
+ }
+
+ @Override
+ public List<String> getCacheFiles() {
+ List<String> list = new ArrayList<String>(1);
+ list.add(this.modelPath + "#" + MODEL_FILE);
+ return list;
+ }
+
+ // Enable multiple languages by specifying the model path. See http://text.sourceforge.net/models-1.5/
+ public DataBag exec(Tuple input) throws IOException
+ {
+ if(input.size() != 1) {
+ throw new IOException();
+ }
+
+ String inputString = input.get(0).toString();
+ if(inputString == null || inputString == "") {
+ return null;
+ }
+ DataBag outBag = bf.newDefaultBag();
+ if(this.tokenizer == null) {
+ String loadFile = CachedFile.getFileName(MODEL_FILE, this.modelPath);;
+ InputStream file = new FileInputStream(loadFile);
+ InputStream buffer = new BufferedInputStream(file);
+ TokenizerModel model = new TokenizerModel(buffer);
+ this.tokenizer = new TokenizerME(model);
+ }
+ String tokens[] = this.tokenizer.tokenize(inputString);
+ for(String token : tokens) {
+ Tuple outTuple = tf.newTuple(token);
+ outBag.add(outTuple);
+ }
+ return outBag;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try
+ {
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.CHARARRAY)
+ {
+ throw new RuntimeException("Expected a CHARARRAY as input, but got a " + inputFieldSchema.toString());
+ }
+
+ Schema tupleSchema = new Schema();
+ tupleSchema.add(new Schema.FieldSchema("token",DataType.CHARARRAY));
+
+ return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
+ .getName()
+ .toLowerCase(), input),
+ tupleSchema,
+ DataType.BAG));
+ }
+ catch (FrontendException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/text/opennlp/TokenizeSimple.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/text/opennlp/TokenizeSimple.java b/datafu-pig/src/main/java/datafu/pig/text/opennlp/TokenizeSimple.java
new file mode 100644
index 0000000..cea48b4
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/text/opennlp/TokenizeSimple.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package datafu.pig.text.opennlp;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import opennlp.tools.tokenize.SimpleTokenizer;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.*;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * The OpenNLP Tokenizers segment an input character sequence into tokens. This one uses the OpenNLP class SimpleTokenizer
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * define TokenizeSimple datafu.pig.text.opennlp.TokenizeSimple();
+ *
+ * -- input:
+ * -- ("I believe the Masons have infiltrated the Apache PMC.")
+ * infoo = LOAD 'input' AS (text:chararray);
+
+ * -- output:
+ * -- ({(I),(believe),(the),(Masons),(have),(infiltrated),(the),(Apache),(PMC),(.)})
+ * outfoo = FOREACH input GENERATE TokenizeSimple(text) as tokens;
+ * }
+ * </pre>
+ */
+public class TokenizeSimple extends EvalFunc<DataBag>
+{
+ private SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
+ private TupleFactory tf = TupleFactory.getInstance();
+ private BagFactory bf = BagFactory.getInstance();
+
+ public DataBag exec(Tuple input) throws IOException
+ {
+ if(input.size() != 1) {
+ throw new IOException();
+ }
+
+ String inputString = input.get(0).toString();
+ if(inputString == null || inputString == "") {
+ return null;
+ }
+
+ DataBag outBag = bf.newDefaultBag();
+ String tokens[] = tokenizer.tokenize(inputString);
+ for(String token : tokens) {
+ Tuple outTuple = tf.newTuple(token);
+ outBag.add(outTuple);
+ }
+ return outBag;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try
+ {
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.CHARARRAY)
+ {
+ throw new RuntimeException("Expected a CHARARRAY as input, but got a " + inputFieldSchema.toString());
+ }
+
+ Schema tupleSchema = new Schema();
+ tupleSchema.add(new Schema.FieldSchema("token",DataType.CHARARRAY));
+
+ return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
+ .getName()
+ .toLowerCase(), input),
+ tupleSchema,
+ DataType.BAG));
+ }
+ catch (FrontendException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/text/opennlp/TokenizeWhitespace.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/text/opennlp/TokenizeWhitespace.java b/datafu-pig/src/main/java/datafu/pig/text/opennlp/TokenizeWhitespace.java
new file mode 100644
index 0000000..8efafb0
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/text/opennlp/TokenizeWhitespace.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package datafu.pig.text.opennlp;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import opennlp.tools.tokenize.WhitespaceTokenizer;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.*;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * The OpenNLP Tokenizers segment an input character sequence into tokens. This one uses the OpenNLP class
+ * WhitespaceTokenizer.
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * define TokenizeWhitespace datafu.pig.text.opennlp.TokenizeWhitespace();
+ *
+ * -- input:
+ * -- ("I believe the Masons have infiltrated the Apache PMC.")
+ * infoo = LOAD 'input' AS (text:chararray);
+
+ * -- output:
+ * -- ({(I),(believe),(the),(Masons),(have),(infiltrated),(the),(Apache),(PMC),(.)})
+ * outfoo = FOREACH input GENERATE TokenizeWhitespace(text) as tokens;
+ * }
+ * </pre>
+ */
+public class TokenizeWhitespace extends EvalFunc<DataBag>
+{
+ private WhitespaceTokenizer tokenizer = WhitespaceTokenizer.INSTANCE;
+ private TupleFactory tf = TupleFactory.getInstance();
+ private BagFactory bf = BagFactory.getInstance();
+
+ public DataBag exec(Tuple input) throws IOException
+ {
+ if(input.size() != 1) {
+ throw new IOException();
+ }
+
+ String inputString = input.get(0).toString();
+ if(inputString == null || inputString == "") {
+ return null;
+ }
+
+ DataBag outBag = bf.newDefaultBag();
+ String tokens[] = tokenizer.tokenize(inputString);
+ for(String token : tokens) {
+ Tuple outTuple = tf.newTuple(token);
+ outBag.add(outTuple);
+ }
+ return outBag;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try
+ {
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.CHARARRAY)
+ {
+ throw new RuntimeException("Expected a CHARARRAY as input, but got a " + inputFieldSchema.toString());
+ }
+
+ Schema tupleSchema = new Schema();
+ tupleSchema.add(new Schema.FieldSchema("token",DataType.CHARARRAY));
+
+ return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
+ .getName()
+ .toLowerCase(), input),
+ tupleSchema,
+ DataType.BAG));
+ }
+ catch (FrontendException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/urls/UserAgentClassify.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/urls/UserAgentClassify.java b/datafu-pig/src/main/java/datafu/pig/urls/UserAgentClassify.java
new file mode 100644
index 0000000..4040fea
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/urls/UserAgentClassify.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.urls;
+
+import datafu.pig.util.SimpleEvalFunc;
+
+/**
+ * Given a user agent string, this UDF classifies clients to 'mobile' and 'desktop'.
+ * Current as of June 2011.
+ */
+public class UserAgentClassify extends SimpleEvalFunc<String>
+{
+
+ public String call(String useragent)
+ {
+ if (useragent.length() < 4)
+ return "desktop"; //
+ String ua=useragent.toLowerCase();
+ if(ua.matches(".*(android|avantgo|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\\/|plucker|pocket|psp|symbian|treo|up\\.(browser|link)|vodafone|wap|windows (ce|phone)|xda|xiino).*")||ua.substring(0,4).matches("1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s\\-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|\\-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw\\-(n|u)|c55\\/|capi|ccwa|cdm\\-|cell|chtm|cldc|cmd\\-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc\\-s|devi|dica|dmob|do(c|p)o|ds(12|\\-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(\\-|_)|g1 u|g560|gene|gf\\-5|g\\-mo|go(\\.w|od)|gr(ad|un)|haie|hcit|hd\\-(m|p|t)|hei\\-|hi(pt|ta)|hp( i|ip)|hs\\-c|ht(c(\\-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i\\-(20|go|ma)|i230|iac( |\\-|\\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\\/)|klon|kp
t |kwc\\-|kyo(c|k)|le(no|xi)|lg( g|\\/(k|l|u)|50|54|e\\-|e\\/|\\-[a-w])|libw|lynx|m1\\-w|m3ga|m50\\/|ma(te|ui|xo)|mc(01|21|ca)|m\\-cr|me(di|rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(\\-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)\\-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|\\-([1-8]|c))|phil|pire|pl(ay|uc)|pn\\-2|po(ck|rt|se)|prox|psio|pt\\-g|qa\\-a|qc(07|12|21|32|60|\\-[2-7]|i\\-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h\\-|oo|p\\-)|sdk\\/|se(c(\\-|0|1)|47|mc|nd|ri)|sgh\\-|shar|sie(\\-|m)|sk\\-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h\\-|v\\-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl\\-|tdg\\-|tel(i|m)|tim\\-|t\\-mo|to(pl|sh)|ts(70|m\\-|m3|m5)|tx\\-9|up(\\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|\\-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(\\-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|xda(\\-|2|g)|yas\\-|your|zeto|zte
\\-"))
+ return "mobile";
+ else
+ return "desktop";
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/urls/package-info.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/urls/package-info.java b/datafu-pig/src/main/java/datafu/pig/urls/package-info.java
new file mode 100644
index 0000000..332beda
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/urls/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * UDFs for processing URLs.
+ */
+package datafu.pig.urls;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/util/AliasableEvalFunc.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/util/AliasableEvalFunc.java b/datafu-pig/src/main/java/datafu/pig/util/AliasableEvalFunc.java
new file mode 100644
index 0000000..ee2c3f3
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/util/AliasableEvalFunc.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.util;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * Makes implementing and using UDFs easier by enabling named parameters.
+ *
+ * <p>
+ * This works by capturing the schema of the input tuple on the front-end and storing it into the UDFContext.
+ * It provides an easy means of referencing the parameters on the back-end to aid in writing schema-based UDFs.
+ * </p>
+ *
+ * <p>
+ * A related class is {@link SimpleEvalFunc}. However they are actually fairly different. The primary purpose of {@link SimpleEvalFunc} is
+ * to skip the boilerplate under the assumption that the arguments in and out are well... simple.
+ * It also assumes that these arguments are in a well-defined positional ordering.
+ * </p>
+ *
+ * <p>
+ * AliasableEvalFunc allows the UDF writer to avoid dealing with all positional assumptions and instead reference fields
+ * by their aliases. This practice allows for more readable code since the alias names should have more meaning
+ * to the reader than the position. This approach is also less error prone since it creates a more explicit contract
+ * for what input the UDF expects and prevents simple mistakes that positional-based UDFs could not easily catch,
+ * such as transposing two fields of the same type. If this contract is violated, say, by attempting to reference
+ * a field that is not present, a meaningful error message may be thrown.
+ * </p>
+ *
+ * <p>
+ * Example: This example computes the monthly payments for mortgages depending on interest rate.
+ * <pre>
+ * {@code
+ * public class MortgagePayment extends AliasableEvalFunc<DataBag> {
+ * ...
+ * public DataBag exec(Tuple input) throws IOException {
+ * DataBag output = BagFactory.getInstance().newDefaultBag();
+ *
+ * Double principal = getDouble(input, "principal"); // get a value from the input tuple by alias
+ * Integer numPayments = getInteger(input, "num_payments");
+ * DataBag interestRates = getBag(input, "interest_rates");
+ *
+ * for (Tuple interestTuple : interestRates) {
+ * Double interest = getDouble(interestTuple, getPrefixedAliasName("interest_rates", "interest_rate")); // get a value from the inner bag tuple by alias
+ * double monthlyPayment = computeMonthlyPayment(principal, numPayments, interest);
+ * output.add(TupleFactory.getInstance().newTuple(monthlyPayment));
+ * }
+ * return output;
+ * }
+ * }
+ * }
+ * </pre>
+ * </p>
+ *
+ * @author wvaughan
+ *
+ * @param <T>
+ */
+public abstract class AliasableEvalFunc<T> extends ContextualEvalFunc<T>
+{
+ private static final String ALIAS_MAP_PROPERTY = "aliasMap";
+
+ private Map<String, Integer> aliasToPosition = null;
+
+ public AliasableEvalFunc() {
+
+ }
+
+ /**
+ * A wrapper method which captures the schema and then calls getOutputSchema
+ */
+ @Override
+ public Schema outputSchema(Schema input) {
+ storeFieldAliases(input);
+ return getOutputSchema(input);
+ }
+
+ /**
+ * Specify the output schema as in {link EvalFunc#outputSchema(Schema)}.
+ *
+ * @param input
+ * @return outputSchema
+ */
+ public abstract Schema getOutputSchema(Schema input);
+
+ @SuppressWarnings("unchecked")
+ private Map<String, Integer> getAliasMap() {
+ return (Map<String, Integer>)getInstanceProperties().get(ALIAS_MAP_PROPERTY);
+ }
+
+ private void setAliasMap(Map<String, Integer> aliases) {
+ getInstanceProperties().put(ALIAS_MAP_PROPERTY, aliases);
+ }
+
+ private void storeFieldAliases(Schema tupleSchema)
+ {
+ Map<String, Integer> aliases = new HashMap<String, Integer>();
+ constructFieldAliases(aliases, tupleSchema, null);
+ log.debug("In instance: "+getInstanceName()+", stored alias map: " + aliases);
+
+ // pass the input schema into the exec function
+ setAliasMap(aliases);
+ }
+
+ private void constructFieldAliases(Map<String, Integer> aliases, Schema tupleSchema, String prefix)
+ {
+ int position = 0;
+ for (Schema.FieldSchema field : tupleSchema.getFields()) {
+ String alias = getPrefixedAliasName(prefix, field.alias);
+ if (field.alias != null && !field.alias.equals("null")) {
+ aliases.put(alias, position);
+ log.debug("In instance: "+getInstanceName()+", stored alias " + alias + " as position " + position);
+ }
+ if (field.schema != null) {
+ constructFieldAliases(aliases, field.schema, alias);
+ }
+ position++;
+ }
+ }
+
+ public String getPrefixedAliasName(String prefix, String alias)
+ {
+ if (alias == null || alias.equals("null")) {
+ if (prefix == null) return "";
+ else return prefix; // ignore the null inner bags/tuples
+ }
+ else return ((prefix == null || prefix.equals("null") || prefix.trim().equals("")) ? "" : prefix+".") + alias; // handle top bag
+ }
+
+ /**
+ * Field aliases are generated from the input schema<br/>
+ * Each alias maps to a bag position<br/>
+ * Inner bags/tuples will have alias of outer.inner.foo
+ *
+ * @return A map of field alias to field position
+ */
+ public Map<String, Integer> getFieldAliases()
+ {
+ Map<String, Integer> aliases = getAliasMap();
+ if (aliases == null) {
+ log.error("Class: " + this.getClass());
+ log.error("Instance name: " + this.getInstanceName());
+ log.error("Properties: " + getContextProperties());
+ throw new RuntimeException("Could not retrieve aliases from properties using " + ALIAS_MAP_PROPERTY);
+ }
+ return aliases;
+ }
+
+ public Integer getPosition(String alias) {
+ if (aliasToPosition == null) {
+ aliasToPosition = getFieldAliases();
+ }
+ return aliasToPosition.get(alias);
+ }
+
+ public Integer getPosition(String prefix, String alias) {
+ return getPosition(getPrefixedAliasName(prefix, alias));
+ }
+
+ public Integer getInteger(Tuple tuple, String alias) throws ExecException {
+ return getInteger(tuple, alias, null);
+ }
+
+ public Integer getInteger(Tuple tuple, String alias, Integer defaultValue) throws ExecException {
+ Integer i = getPosition(alias);
+ if (i == null) throw new FieldNotFound("Attempt to reference unknown alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
+ if (i >= tuple.size()) throw new FieldNotFound("Attempt to reference outside of tuple for alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
+ Number number = (Number)tuple.get(i);
+ if (number == null) return defaultValue;
+ return number.intValue();
+ }
+
+ public Long getLong(Tuple tuple, String alias) throws ExecException {
+ return getLong(tuple, alias, null);
+ }
+
+ public Long getLong(Tuple tuple, String alias, Long defaultValue) throws ExecException {
+ Integer i = getPosition(alias);
+ if (i == null) throw new FieldNotFound("Attempt to reference unknown alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
+ if (i >= tuple.size()) throw new FieldNotFound("Attempt to reference outside of tuple for alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
+ Number number = (Number)tuple.get(i);
+ if (number == null) return defaultValue;
+ return number.longValue();
+ }
+
+ public Float getFloat(Tuple tuple, String alias) throws ExecException {
+ return getFloat(tuple, alias, null);
+ }
+
+ public Float getFloat(Tuple tuple, String alias, Float defaultValue) throws ExecException {
+ Integer i = getPosition(alias);
+ if (i == null) throw new FieldNotFound("Attempt to reference unknown alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
+ if (i >= tuple.size()) throw new FieldNotFound("Attempt to reference outside of tuple for alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
+ Number number = (Number)tuple.get(i);
+ if (number == null) return defaultValue;
+ return number.floatValue();
+ }
+
+ public Double getDouble(Tuple tuple, String alias) throws ExecException {
+ return getDouble(tuple, alias, null);
+ }
+
+ public Double getDouble(Tuple tuple, String alias, Double defaultValue) throws ExecException {
+ Integer i = getPosition(alias);
+ if (i == null) throw new FieldNotFound("Attempt to reference unknown alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
+ if (i >= tuple.size()) throw new FieldNotFound("Attempt to reference outside of tuple for alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
+ Number number = (Number)tuple.get(i);
+ if (number == null) return defaultValue;
+ return number.doubleValue();
+ }
+
+ public String getString(Tuple tuple, String alias) throws ExecException {
+ return getString(tuple, alias, null);
+ }
+
+ public String getString(Tuple tuple, String alias, String defaultValue) throws ExecException {
+ Integer i = getPosition(alias);
+ if (i == null) throw new FieldNotFound("Attempt to reference unknown alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
+ if (i >= tuple.size()) throw new FieldNotFound("Attempt to reference outside of tuple for alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
+ String s = (String)tuple.get(i);
+ if (s == null) return defaultValue;
+ return s;
+ }
+
+ public Boolean getBoolean(Tuple tuple, String alias) throws ExecException {
+ Integer i = getPosition(alias);
+ if (i == null) throw new FieldNotFound("Attempt to reference unknown alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
+ if (i >= tuple.size()) throw new FieldNotFound("Attempt to reference outside of tuple for alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
+ return (Boolean)tuple.get(i);
+ }
+
+ public DataBag getBag(Tuple tuple, String alias) throws ExecException {
+ Integer i = getPosition(alias);
+ if (i == null) throw new FieldNotFound("Attempt to reference unknown alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
+ if (i >= tuple.size()) throw new FieldNotFound("Attempt to reference outside of tuple for alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
+ return (DataBag)tuple.get(i);
+ }
+
+ public Object getObject(Tuple tuple, String alias) throws ExecException {
+ Integer i = getPosition(alias);
+ if (i == null) throw new FieldNotFound("Attempt to reference unknown alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
+ if (i >= tuple.size()) throw new FieldNotFound("Attempt to reference outside of tuple for alias: "+alias+"\n Instance Properties: "+getInstanceProperties());
+ return tuple.get(i);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/util/Assert.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/util/Assert.java b/datafu-pig/src/main/java/datafu/pig/util/Assert.java
new file mode 100644
index 0000000..a258915
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/util/Assert.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.util;
+
+/**
+ * Assert has been renamed to AssertUDF.
+ *
+ * This class is provided for backward compatibility.
+ *
+ * @deprecated Use {@link AssertUDF} instead.
+ */
+ @Deprecated
+public class Assert extends AssertUDF
+{
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/util/AssertUDF.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/util/AssertUDF.java b/datafu-pig/src/main/java/datafu/pig/util/AssertUDF.java
new file mode 100644
index 0000000..16f9247
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/util/AssertUDF.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.util;
+
+import java.io.IOException;
+
+import org.apache.pig.FilterFunc;
+import org.apache.pig.data.Tuple;
+
+/**
+ * Filter function which asserts that a value is true.
+ *
+ * <p>
+ * Unfortunately, the Pig interpreter doesn't recognize boolean expressions nested as function
+ * arguments, so this uses C-style booleans. That is, the first argument should be
+ * an integer. 0 is interpreted as "false", and anything else is considered "true".
+ * The function will cause the Pig script to fail if a "false" value is encountered.
+ * </p>
+ *
+ * <p>
+ * There is a unary and a binary version. The unary version just takes a boolean, and throws out a generic exception message when the
+ * assertion is violated. The binary version takes a String as a second argument and throws that out when the assertion
+ * is violated.
+ * </p>
+ *
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * FILTER members BY AssertUDF( (member_id >= 0 ? 1 : 0), 'Doh! Some member ID is negative.' );
+ * }
+ * </pre>
+ * </p>
+ */
+public class AssertUDF extends FilterFunc
+{
+ @Override
+ public Boolean exec(Tuple tuple)
+ throws IOException
+ {
+ if ((Integer) tuple.get(0) == 0) {
+ if (tuple.size() > 1) {
+ throw new IOException("Assertion violated: " + tuple.get(1).toString());
+ }
+ else {
+ throw new IOException("Assertion violated. What assertion, I do not know, but it was officially violated.");
+ }
+ }
+ else {
+ return true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/util/BoolToInt.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/util/BoolToInt.java b/datafu-pig/src/main/java/datafu/pig/util/BoolToInt.java
new file mode 100644
index 0000000..5ec8d3a
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/util/BoolToInt.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.util;
+
+/**
+ * UDF which converts a Boolean to an Integer.
+ */
+public class BoolToInt extends SimpleEvalFunc<Integer>
+{
+ public Integer call(Boolean val)
+ {
+ return (val == null || !val) ? 0 : 1;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/util/Coalesce.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/util/Coalesce.java b/datafu-pig/src/main/java/datafu/pig/util/Coalesce.java
new file mode 100644
index 0000000..f8e25f4
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/util/Coalesce.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.util;
+
+import java.io.IOException;
+
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
+
+/**
+ * Returns the first non-null value from a tuple, just like {@link <a href="http://msdn.microsoft.com/en-us/library/ms190349.aspx" target="_blank">COALESCE</a>} in SQL.
+ *
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ *
+ * define COALESCE datafu.pig.util.COALESCE();
+
+ * -- input: 1,2,3,NULL,4,NULL,5
+ * input = LOAD 'input' AS (val:int);
+ *
+ * -- produces: 1,2,3,99,4,99,5
+ * coalesced = FOREACH input GENERATE COALESCE(val,99);
+ *
+ * }
+ * </pre>
+ * </p>
+ *
+ * @author "Matthew Hayes <mh...@linkedin.com>"
+ *
+ */
+public class Coalesce extends AliasableEvalFunc<Object>
+{
+ private boolean strict;
+
+ private static String STRICT_OPTION = "strict";
+ private static String LAZY_OPTION = "lazy";
+
+ public Coalesce()
+ {
+ strict = true;
+ }
+
+ public Coalesce(String option)
+ {
+ if (option.equals(STRICT_OPTION))
+ {
+ strict = true;
+ }
+ else if (option.equals(LAZY_OPTION))
+ {
+ strict = false;
+ }
+ else
+ {
+ throw new IllegalArgumentException("Unexpected option: " + option);
+ }
+ }
+
+ @Override
+ public Object exec(Tuple input) throws IOException
+ {
+ if (input == null || input.size() == 0)
+ {
+ return null;
+ }
+
+ Byte type = (Byte)getInstanceProperties().get("type");
+
+ for (Object o : input)
+ {
+ if (o != null)
+ {
+ if (strict)
+ {
+ return o;
+ }
+ else
+ {
+ try
+ {
+ switch (type)
+ {
+ case DataType.INTEGER:
+ return DataType.toInteger(o);
+ case DataType.LONG:
+ return DataType.toLong(o);
+ case DataType.DOUBLE:
+ return DataType.toDouble(o);
+ case DataType.FLOAT:
+ return DataType.toFloat(o);
+ default:
+ return o;
+ }
+ }
+ catch (Exception e)
+ {
+ DataFuException dfe = new DataFuException(e.getMessage(),e);
+ dfe.setData(o);
+ dfe.setFieldAliases(getFieldAliases());
+ throw dfe;
+ }
+ }
+ }
+ }
+
+ return null;
+ }
+
+ @Override
+ public Schema getOutputSchema(Schema input)
+ {
+ if (input.getFields().size() == 0)
+ {
+ throw new RuntimeException("Expected at least one parameter");
+ }
+
+ Byte outputType = null;
+ int pos = 0;
+ for (FieldSchema field : input.getFields())
+ {
+ if (DataType.isSchemaType(field.type))
+ {
+ throw new RuntimeException(String.format("Not supported on schema types. Found %s in position %d.",DataType.findTypeName(field.type),pos));
+ }
+
+ if (DataType.isComplex(field.type))
+ {
+ throw new RuntimeException(String.format("Not supported on complex types. Found %s in position %d.",DataType.findTypeName(field.type),pos));
+ }
+
+ if (!DataType.isUsableType(field.type))
+ {
+ throw new RuntimeException(String.format("Not a usable type. Found %s in position %d.",DataType.findTypeName(field.type),pos));
+ }
+
+ if (outputType == null)
+ {
+ outputType = field.type;
+ }
+ else if (!outputType.equals(field.type))
+ {
+ if (strict)
+ {
+ throw new RuntimeException(String.format("Expected all types to be equal, but found '%s' in position %d. First element has type '%s'. "
+ + "If you'd like to attempt merging types, use the '%s' option, as '%s' is the default.",
+ DataType.findTypeName(field.type),pos,DataType.findTypeName((byte)outputType),LAZY_OPTION,STRICT_OPTION));
+ }
+ else
+ {
+ byte merged = DataType.mergeType(outputType, field.type);
+ if (merged == DataType.ERROR)
+ {
+ throw new RuntimeException(String.format("Expected all types to be equal, but found '%s' in position %d, where output type is '%s', and types could not be merged.",
+ DataType.findTypeName(field.type),pos,DataType.findTypeName((byte)outputType)));
+ }
+ outputType = merged;
+ }
+ }
+
+ pos++;
+ }
+
+ getInstanceProperties().put("type", outputType);
+
+ return new Schema(new Schema.FieldSchema("item",outputType));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/util/ContextualEvalFunc.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/util/ContextualEvalFunc.java b/datafu-pig/src/main/java/datafu/pig/util/ContextualEvalFunc.java
new file mode 100644
index 0000000..c534b77
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/util/ContextualEvalFunc.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.util;
+
+import java.util.Properties;
+
+import org.apache.pig.EvalFunc;
+import org.apache.pig.impl.util.UDFContext;
+
+/**
+ * An abstract class which enables UDFs to store instance properties
+ * on the front end which will be available on the back end.
+ * For example, properties may be set in the call to outputSchema(),
+ * which will be available when exec() is called.
+ *
+ * @param <T>
+ */
+public abstract class ContextualEvalFunc<T> extends EvalFunc<T>
+{
+ private String instanceName;
+
+ @Override
+ public void setUDFContextSignature(String signature) {
+ setInstanceName(signature);
+ }
+
+ /**
+ * Helper method to return the context properties for this class
+ *
+ * @return context properties
+ */
+ protected Properties getContextProperties() {
+ UDFContext context = UDFContext.getUDFContext();
+ Properties properties = context.getUDFProperties(this.getClass());
+ return properties;
+ }
+
+ /**
+ * Helper method to return the context properties for this instance of this class
+ *
+ * @return instances properties
+ */
+ protected Properties getInstanceProperties() {
+ Properties contextProperties = getContextProperties();
+ if (!contextProperties.containsKey(getInstanceName())) {
+ contextProperties.put(getInstanceName(), new Properties());
+ }
+ return (Properties)contextProperties.get(getInstanceName());
+ }
+
+ /**
+ *
+ * @return the name of this instance corresponding to the UDF Context Signature
+ * @see #setUDFContextSignature(String)
+ */
+ protected String getInstanceName() {
+ if (instanceName == null) {
+ throw new RuntimeException("Instance name is null. This should not happen unless UDFContextSignature was not set.");
+ }
+ return instanceName;
+ }
+
+ private void setInstanceName(String instanceName) {
+ this.instanceName = instanceName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/util/DataFuException.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/util/DataFuException.java b/datafu-pig/src/main/java/datafu/pig/util/DataFuException.java
new file mode 100644
index 0000000..0066aa8
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/util/DataFuException.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.util;
+
+import java.util.Map;
+
+public class DataFuException extends RuntimeException
+{
+ private static final long serialVersionUID = 1L;
+ private Map<String, Integer> fieldAliases;
+ private Object data;
+
+ public DataFuException()
+ {
+ super();
+ }
+
+ public DataFuException(String message)
+ {
+ super(message);
+ }
+
+ public DataFuException(String message, Throwable cause)
+ {
+ super(message, cause);
+ }
+
+ public DataFuException(Throwable cause)
+ {
+ super(cause);
+ }
+
+ /**
+ * Gets field aliases for a UDF which may be relevant to this exception.
+ *
+ * @return field aliases
+ */
+ public Map<String, Integer> getFieldAliases()
+ {
+ return fieldAliases;
+ }
+
+ /**
+ * Gets data relevant to this exception.
+ *
+ * @return data
+ */
+ public Object getData()
+ {
+ return data;
+ }
+
+ /**
+ * Sets field aliases for a UDF which may be relevant to this exception.
+ *
+ * @param fieldAliases
+ */
+ public void setFieldAliases(Map<String, Integer> fieldAliases)
+ {
+ this.fieldAliases = fieldAliases;
+ }
+
+ /**
+ * Sets data relevant to this exception.
+ * @param data
+ */
+ public void setData(Object data)
+ {
+ this.data = data;
+ }
+
+ @Override
+ public String toString()
+ {
+ String s = getClass().getName();
+ String message = getLocalizedMessage();
+
+ StringBuilder result = new StringBuilder(s);
+
+ if (message != null)
+ {
+ result.append(": ");
+ result.append(message);
+ }
+
+ if (getFieldAliases() != null)
+ {
+ result.append("\nAliases:");
+ for (String alias : getFieldAliases().keySet())
+ {
+ result.append("\n");
+ result.append(alias != null && alias.length() > 0 ? alias : "???");
+ }
+ }
+
+ if (getData() != null)
+ {
+ result.append("\nData:");
+ result.append("\n");
+ result.append(data.toString());
+ }
+
+ return result.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/util/FieldNotFound.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/util/FieldNotFound.java b/datafu-pig/src/main/java/datafu/pig/util/FieldNotFound.java
new file mode 100644
index 0000000..d624007
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/util/FieldNotFound.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.util;
+
+import org.apache.pig.backend.executionengine.ExecException;
+
+/**
+ * Thrown by {see AliasableEvalFunc} when attempting to access an unknown field by name.
+ *
+ * @author wvaughan
+ *
+ */
+public class FieldNotFound extends ExecException
+{
+ private static final long serialVersionUID = 1L;
+
+ public FieldNotFound() {
+ super();
+ }
+
+ public FieldNotFound(String message) {
+ super(message);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/util/In.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/util/In.java b/datafu-pig/src/main/java/datafu/pig/util/In.java
new file mode 100644
index 0000000..0667914
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/util/In.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.util;
+
+/**
+ * In has been renamed to InUDF.
+ *
+ * This class is provided for backward compatibility.
+ *
+ * @deprecated Use {@link InUDF} instead.
+ */
+ @Deprecated
+public class In extends InUDF
+{
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/util/InUDF.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/util/InUDF.java b/datafu-pig/src/main/java/datafu/pig/util/InUDF.java
new file mode 100644
index 0000000..5057285
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/util/InUDF.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.util;
+
+import java.io.IOException;
+
+import org.apache.pig.FilterFunc;
+import org.apache.pig.data.Tuple;
+
+/**
+ * Similar to the SQL IN function, this function provides a convenient way to filter
+ * using a logical disjunction over many values.
+ * Returns true when the first value of the tuple is contained within the remainder of the tuple.
+ *
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * define In datafu.pig.util.InUDF();
+ * -- cars: (alice, red), (bob, blue), (charlie, green), (dave, red);
+ * cars = LOAD cars AS (owner:chararray, color:chararray);
+ *
+ * -- cars: (alice, red), (bob, blue), (dave, red);
+ * red_blue_cars = FILTER cars BY In(color, 'red', 'blue');
+ *
+ * }</pre>
+ * </p>
+ *
+ * @author wvaughan
+ *
+ */
+public class InUDF extends FilterFunc
+{
+
+ @Override
+ public Boolean exec(Tuple input) throws IOException
+ {
+ Object o = input.get(0);
+ Boolean match = false;
+ if (o != null) {
+ for (int i=1; i<input.size() && !match; i++) {
+ match = match || o.equals(input.get(i));
+ }
+ }
+ return match;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/util/IntToBool.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/util/IntToBool.java b/datafu-pig/src/main/java/datafu/pig/util/IntToBool.java
new file mode 100644
index 0000000..d00e297
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/util/IntToBool.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.util;
+
+/**
+ * UDF which converts an Integer to a Boolean.
+ */
+public class IntToBool extends SimpleEvalFunc<Boolean>
+{
+ public Boolean call(Integer val)
+ {
+ return (val == null || val == 0) ? false : true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/util/SimpleEvalFunc.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/util/SimpleEvalFunc.java b/datafu-pig/src/main/java/datafu/pig/util/SimpleEvalFunc.java
new file mode 100644
index 0000000..2d262b4
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/util/SimpleEvalFunc.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.util;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Type;
+
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+
+/**
+ Uses reflection to makes writing simple wrapper Pig UDFs easier.
+
+ For example, writing a simple string trimming UDF might look like
+ this:
+ <pre>
+ {@code
+ public class TRIM extends EvalFunc<String>
+ {
+ public String exec(Tuple input) throws IOException
+ {
+ if (input.size() != 1)
+ throw new IllegalArgumentException("requires a parameter");
+
+ try {
+ Object o = input.get(0);
+ if (!(o instanceof String))
+ throw new IllegalArgumentException("expected a string");
+
+ String str = (String)o;
+ return (str == null) ? null : str.trim();
+ }
+ catch (Exception e) {
+ throw WrappedIOException.wrap("error...", e);
+ }
+ }
+ }
+ }
+ </pre>
+ There is a lot of boilerplate to check the number of arguments and
+ the parameter types in the tuple.
+
+ Instead, with this class, you can derive from SimpleEvalFunc and
+ create a <code>call()</code> method (not exec!), just specifying the
+ arguments as a regular function. The class handles all the argument
+ checking and exception wrapping for you. So your code would be:
+ <pre>
+ {@code
+ public class TRIM2 extends SimpleEvalFunc<String>
+ {
+ public String call(String s)
+ {
+ return (s != null) ? s.trim() : null;
+ }
+ }
+ }
+ </pre>
+
+ An example of this UDF in action with Pig:
+ <pre>
+ {@code
+ grunt> a = load 'test' as (x:chararray, y:chararray); dump a;
+ (1 , 2)
+
+ grunt> b = foreach a generate TRIM2(x); dump b;
+ (1)
+
+ grunt> c = foreach a generate TRIM2((int)x); dump c;
+ datafu.pig.util.TRIM2(java.lang.String): argument type
+ mismatch [#1]; expected java.lang.String, got java.lang.Integer
+
+ grunt> d = foreach a generate TRIM2(x, y); dump d;
+ datafu.pig.util.TRIM2(java.lang.String): got 2 arguments, expected 1.
+ }
+ </pre>
+
+*/
+
+public abstract class SimpleEvalFunc<T> extends EvalFunc<T>
+{
+ // TODO Add support for other UDF types (e.g., FilterFunc)
+ // TODO Algebraic EvalFuncs
+
+ Method m = null;
+
+ public SimpleEvalFunc()
+ {
+ for (Method method : this.getClass().getMethods()) {
+ if (method.getName() == "call")
+ m = method;
+ }
+ if (m == null)
+ throw new IllegalArgumentException(String.format("%s: couldn't find call() method in UDF.", getClass().getName()));
+ }
+
+ // Pig can't get the return type via reflection (as getReturnType normally tries to do), so give it a hand
+ @Override
+ public Type getReturnType()
+ {
+ return m.getReturnType();
+ }
+
+ private String _method_signature()
+ {
+ StringBuilder sb = new StringBuilder(getClass().getName());
+ Class<?> pvec[] = m.getParameterTypes();
+
+ sb.append("(");
+ for (int i=0; i < pvec.length; i++) {
+ if (i > 0)
+ sb.append(", ");
+ sb.append(String.format("%s", pvec[i].getName()));
+ }
+ sb.append(")");
+
+ return sb.toString();
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public T exec(Tuple input) throws IOException
+ {
+ @SuppressWarnings("rawtypes")
+ Class pvec[] = m.getParameterTypes();
+
+ if (input == null || input.size() == 0)
+ return null;
+
+ // check right number of arguments
+ if (input.size() != pvec.length)
+ throw new IOException(String.format("%s: got %d arguments, expected %d.", _method_signature(), input.size(), pvec.length));
+
+ // pull and check argument types
+ Object[] args = new Object[input.size()];
+ for (int i=0; i < pvec.length; i++) {
+ Object o = input.get(i);
+ try {
+ o = pvec[i].cast(o);
+ }
+ catch (ClassCastException e) {
+ throw new IOException(String.format("%s: argument type mismatch [#%d]; expected %s, got %s", _method_signature(), i+1,
+ pvec[i].getName(), o.getClass().getName()));
+ }
+ args[i] = o;
+ }
+
+ try {
+ return (T) m.invoke(this, args);
+ }
+ catch (Exception e) {
+ throw new IOException(String.format("%s: caught exception processing input.", _method_signature()), e);
+ }
+ }
+
+ /**
+ * Override outputSchema so we can verify the input schema at pig compile time, instead of runtime
+ * @param inputSchema input schema
+ * @return call to super.outputSchema in case schema was defined elsewhere
+ */
+ @Override
+ public Schema outputSchema(Schema inputSchema)
+ {
+ if (inputSchema == null) {
+ throw new IllegalArgumentException(String.format("%s: null schema passed to %s", _method_signature(), getClass().getName()));
+ }
+
+ // check correct number of arguments
+ @SuppressWarnings("rawtypes")
+ Class parameterTypes[] = m.getParameterTypes();
+ if (inputSchema.size() != parameterTypes.length) {
+ throw new IllegalArgumentException(String.format("%s: got %d arguments, expected %d.",
+ _method_signature(),
+ inputSchema.size(),
+ parameterTypes.length));
+ }
+
+ // check type for each argument
+ for (int i=0; i < parameterTypes.length; i++) {
+ try {
+ byte inputType = inputSchema.getField(i).type;
+ byte parameterType = DataType.findType(parameterTypes[i]);
+ if (inputType != parameterType) {
+ throw new IllegalArgumentException(String.format("%s: argument type mismatch [#%d]; expected %s, got %s",
+ _method_signature(),
+ i+1,
+ DataType.findTypeName(parameterType),
+ DataType.findTypeName(inputType)));
+ }
+ }
+ catch (FrontendException fe) {
+ throw new IllegalArgumentException(String.format("%s: Problem with input schema: ", _method_signature(), inputSchema), fe);
+ }
+ }
+
+ // delegate to super to determine the actual outputSchema (if specified)
+ return super.outputSchema(inputSchema);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/util/TransposeTupleToBag.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/util/TransposeTupleToBag.java b/datafu-pig/src/main/java/datafu/pig/util/TransposeTupleToBag.java
new file mode 100644
index 0000000..f8a39df
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/util/TransposeTupleToBag.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.util;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
+
+/**
+ * Performs a transpose on a tuple, resulting in a bag of key, value fields where
+ * the key is the column name and the value is the value of that column in the tuple.
+ *
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ *
+ * define TransposeTupleToBag datafu.pig.util.TransposeTupleToBag();
+
+ * -- input: 1,10,11,12
+ * input = LOAD 'input' AS (id:int,val1:int,val2:int,val3:int);
+ *
+ * -- produces: 1,{("val1",10),("val2",11),("val3",12)}
+ * output = FOREACH input GENERATE id, TransposeTupleToBag(val1 .. val3);
+ *
+ * }
+ * </pre>
+ * </p>
+ *
+ * @author "William Vaughan <wv...@linkedin.com>"
+ *
+ */
+public class TransposeTupleToBag extends AliasableEvalFunc<DataBag>
+{
+ private final String TRANSPOSE_TYPE = "TRANSPOSE_TYPE";
+
+ @Override
+ public Schema getOutputSchema(Schema input)
+ {
+ try
+ {
+ // require that every field in the input has the same type
+ Byte type = null;
+ for (FieldSchema fieldSchema : input.getFields()) {
+ if (type == null) {
+ type = fieldSchema.type;
+ } else {
+ if (type != fieldSchema.type) {
+ throw new RuntimeException(
+ String.format("Expected all input types to match. Got both %s and %s.",
+ DataType.findTypeName(type.byteValue()), DataType.findTypeName(fieldSchema.type)));
+ }
+ }
+ }
+ getInstanceProperties().put(TRANSPOSE_TYPE, type);
+
+ Schema outputTupleSchema = new Schema();
+ outputTupleSchema.add(new Schema.FieldSchema("key", DataType.CHARARRAY));
+ outputTupleSchema.add(new Schema.FieldSchema("value", type));
+ return new Schema(new Schema.FieldSchema(
+ getSchemaName(this.getClass().getName().toLowerCase(), input),
+ outputTupleSchema,
+ DataType.BAG));
+ }
+ catch (FrontendException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public DataBag exec(Tuple input) throws IOException
+ {
+ // initialize a reverse mapping
+ HashMap<Integer, String> positionToAlias = new HashMap<Integer, String>();
+ for (String alias : getFieldAliases().keySet()) {
+ positionToAlias.put(getFieldAliases().get(alias), alias);
+ }
+ DataBag output = BagFactory.getInstance().newDefaultBag();
+ for (int i=0; i<input.size(); i++) {
+ Tuple tuple = TupleFactory.getInstance().newTuple();
+ tuple.append(positionToAlias.get(i));
+ tuple.append(input.get(i));
+ output.add(tuple);
+ }
+ return output;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/util/package-info.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/util/package-info.java b/datafu-pig/src/main/java/datafu/pig/util/package-info.java
new file mode 100644
index 0000000..cce46a0
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/util/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Other useful utilities.
+ */
+package datafu.pig.util;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/PigTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/PigTests.java b/datafu-pig/src/test/java/datafu/test/pig/PigTests.java
new file mode 100644
index 0000000..d1d6fcc
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/PigTests.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig;
+
+import static org.testng.Assert.*;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.metrics.jvm.JvmMetrics;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.pigunit.PigTest;
+import org.apache.pig.tools.parameters.ParseException;
+
+public abstract class PigTests
+{
+ @org.testng.annotations.BeforeClass
+ public void beforeClass()
+ {
+ // TODO make it configurable whether this happens, for travis-ci we can't spam the logs so much,
+ // however otherwise it is useful to see the errors
+ Logger.getRootLogger().removeAllAppenders();
+ Logger.getLogger(JvmMetrics.class).setLevel(Level.OFF);
+ }
+
+ @org.testng.annotations.BeforeMethod
+ public void beforeMethod(Method method)
+ {
+ System.out.println("\n*** Running " + method.getName() + " ***");
+ }
+
+ protected String[] getDefaultArgs()
+ {
+ String[] args = {
+ getDataDirParam()
+ };
+ return args;
+ }
+
+ protected List<String> getDefaultArgsAsList()
+ {
+ String[] args = getDefaultArgs();
+ List<String> argsList = new ArrayList<String>(args.length);
+ for (String arg : args)
+ {
+ argsList.add(arg);
+ }
+ return argsList;
+ }
+
+ protected PigTest createPigTestFromString(String str, String... args) throws IOException
+ {
+ return createPigTest(str.split("\n"),args);
+ }
+
+ protected PigTest createPigTest(String[] lines, String... args) throws IOException
+ {
+ // append args to list of default args
+ List<String> theArgs = getDefaultArgsAsList();
+ for (String arg : args)
+ {
+ theArgs.add(arg);
+ }
+
+ for (String arg : theArgs)
+ {
+ String[] parts = arg.split("=",2);
+ if (parts.length == 2)
+ {
+ for (int i=0; i<lines.length; i++)
+ {
+ lines[i] = lines[i].replaceAll(Pattern.quote("$" + parts[0]), parts[1]);
+ }
+ }
+ }
+
+ return new PigTest(lines);
+ }
+
+ protected PigTest createPigTest(String scriptPath, String... args) throws IOException
+ {
+ return createPigTest(getLinesFromFile(scriptPath), args);
+ }
+
+ protected String getDataDirParam()
+ {
+ return "DATA_DIR=" + getDataPath();
+ }
+
+ protected String getDataPath()
+ {
+ if (System.getProperty("datafu.data.dir") != null)
+ {
+ return System.getProperty("datafu.data.dir");
+ }
+ else
+ {
+ return new File(System.getProperty("user.dir"), "data").getAbsolutePath();
+ }
+ }
+
+ protected String getJarPath()
+ {
+ String jarDir = null;
+
+ if (System.getProperty("datafu.jar.dir") != null)
+ {
+ jarDir = System.getProperty("datafu.jar.dir");
+ }
+ else
+ {
+ jarDir = new File(System.getProperty("user.dir"), "build/libs").getAbsolutePath();
+ }
+
+ File userDir = new File(jarDir);
+
+ String[] files = userDir.list(new FilenameFilter() {
+
+ @Override
+ public boolean accept(File dir, String name)
+ {
+ return name.endsWith(".jar") && !name.contains("sources") && !name.contains("javadoc");
+ }
+
+ });
+
+ if (files == null || files.length == 0)
+ {
+ throw new RuntimeException("Could not find JAR file");
+ }
+ else if (files.length > 1)
+ {
+ StringBuilder sb = new StringBuilder();
+ for (String file : files)
+ {
+ sb.append(file);
+ sb.append(",");
+ }
+ throw new RuntimeException("Found more JAR files than expected: " + sb.substring(0, sb.length()-1));
+ }
+
+ return userDir.getAbsolutePath() + "/" + files[0];
+ }
+
+ protected List<Tuple> getLinesForAlias(PigTest test, String alias) throws IOException, ParseException
+ {
+ return getLinesForAlias(test,alias,true);
+ }
+
+ protected List<Tuple> getLinesForAlias(PigTest test, String alias, boolean logValues) throws IOException, ParseException
+ {
+ Iterator<Tuple> tuplesIterator = test.getAlias(alias);
+ List<Tuple> tuples = new ArrayList<Tuple>();
+ if (logValues)
+ {
+ System.out.println(String.format("Values for %s: ", alias));
+ }
+ while (tuplesIterator.hasNext())
+ {
+ Tuple tuple = tuplesIterator.next();
+ if (logValues)
+ {
+ System.out.println(tuple.toString());
+ }
+ tuples.add(tuple);
+ }
+ return tuples;
+ }
+
+ protected void writeLinesToFile(String fileName, String... lines) throws IOException
+ {
+ File inputFile = deleteIfExists(getFile(fileName));
+ writeLinesToFile(inputFile, lines);
+ }
+
+ protected void writeLinesToFile(File file, String[] lines) throws IOException
+ {
+ FileWriter writer = new FileWriter(file);
+ for (String line : lines)
+ {
+ writer.write(line + "\n");
+ }
+ writer.close();
+ }
+
+ protected void assertOutput(PigTest test, String alias, String... expected) throws IOException, ParseException
+ {
+ List<Tuple> tuples = getLinesForAlias(test, alias);
+ assertEquals(expected.length, tuples.size());
+ int i=0;
+ for (String e : expected)
+ {
+ assertEquals(tuples.get(i++).toString(), e);
+ }
+ }
+
+ protected File deleteIfExists(File file)
+ {
+ if (file.exists())
+ {
+ file.delete();
+ }
+ return file;
+ }
+
+ protected File getFile(String fileName)
+ {
+ return new File(System.getProperty("user.dir"), fileName).getAbsoluteFile();
+ }
+
+ /**
+ * Gets the lines from a given file.
+ *
+ * @param relativeFilePath The path relative to the datafu-tests project.
+ * @return The lines from the file
+ * @throws IOException
+ */
+ protected String[] getLinesFromFile(String relativeFilePath) throws IOException
+ {
+ // assume that the working directory is the datafu-tests project
+ File file = new File(System.getProperty("user.dir"), relativeFilePath).getAbsoluteFile();
+ BufferedInputStream content = new BufferedInputStream(new FileInputStream(file));
+ Object[] lines = IOUtils.readLines(content).toArray();
+ String[] result = new String[lines.length];
+ for (int i=0; i<lines.length; i++)
+ {
+ result[i] = (String)lines[i];
+ }
+ return result;
+ }
+}
[17/19] DATAFU-27 Migrate build system to Gradle
Posted by wv...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/sampling/ReservoirSample.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/sampling/ReservoirSample.java b/datafu-pig/src/main/java/datafu/pig/sampling/ReservoirSample.java
new file mode 100644
index 0000000..48deaad
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/sampling/ReservoirSample.java
@@ -0,0 +1,327 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.sampling;
+
+import java.io.IOException;
+
+import org.apache.pig.AccumulatorEvalFunc;
+import org.apache.pig.Algebraic;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.builtin.Nondeterministic;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * Performs a simple random sample using an in-memory reservoir to produce
+ * a uniformly random sample of a given size.
+ *
+ * <p>
+ * This is similar to {@link SimpleRandomSample}, however it is guaranteed to produce
+ * a sample of the given size. This comes at the cost of scalability.
+ * {@link SimpleRandomSample} produces a sample of the desired size with likelihood of 99.99%,
+ * while using less internal storage. ReservoirSample on the other hand uses internal storage
+ * with size equaling the desired sample to guarantee the exact sample size.
+ * </p>
+ *
+ * <p>
+ * This algebraic implementation is backed by a heap and maintains the original roll in order
+ * to compensate for skew.
+ * </p>
+ *
+ * @author wvaughan
+ *
+ */
+@Nondeterministic
+public class ReservoirSample extends AccumulatorEvalFunc<DataBag> implements Algebraic
+{
+ protected Integer numSamples;
+
+ private Reservoir reservoir;
+
+ protected ScoredTuple.ScoreGenerator scoreGen;
+
+ private Reservoir getReservoir()
+ {
+ if (reservoir == null) {
+ reservoir = new Reservoir(this.numSamples);
+ }
+ return reservoir;
+ }
+
+ public ReservoirSample(String numSamples)
+ {
+ this.numSamples = Integer.parseInt(numSamples);
+ }
+
+ protected ScoredTuple.ScoreGenerator getScoreGenerator()
+ {
+ if(this.scoreGen == null)
+ {
+ this.scoreGen = new ScoredTuple.PureRandomScoreGenerator();
+ }
+ return this.scoreGen;
+ }
+
+ @Override
+ public void accumulate(Tuple input) throws IOException
+ {
+ DataBag samples = (DataBag) input.get(0);
+ ScoredTuple.ScoreGenerator scoreGen = getScoreGenerator();
+ for (Tuple sample : samples) {
+ getReservoir().consider(new ScoredTuple(scoreGen.generateScore(sample), sample));
+ }
+ }
+
+ @Override
+ public void cleanup()
+ {
+ this.reservoir.clear();
+ }
+
+ @Override
+ public DataBag getValue()
+ {
+ DataBag output = BagFactory.getInstance().newDefaultBag();
+ for (ScoredTuple sample : getReservoir()) {
+ output.add(sample.getTuple());
+ }
+ return output;
+ }
+
+ @Override
+ public DataBag exec(Tuple input) throws IOException
+ {
+ DataBag samples = (DataBag)input.get(0);
+ if (samples.size() <= numSamples) {
+ return samples;
+ }
+ else
+ {
+ return super.exec(input);
+ }
+ }
+
+ @Override
+ public Schema outputSchema(Schema input) {
+ try {
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.BAG) {
+ throw new RuntimeException("Expected a BAG as input");
+ }
+
+ return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
+ inputFieldSchema.schema, DataType.BAG));
+ } catch (FrontendException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ }
+
+ String param = null;
+ private String getParam()
+ {
+ if (param == null) {
+ if (numSamples != null) {
+ param = String.format("('%d')", numSamples);
+ } else {
+ param = "";
+ }
+ }
+ return param;
+ }
+
+ @Override
+ public String getInitial() {
+ return Initial.class.getName()+getParam();
+ }
+
+ @Override
+ public String getIntermed() {
+ return Intermediate.class.getName()+getParam();
+ }
+
+ @Override
+ public String getFinal() {
+ return Final.class.getName()+getParam();
+ }
+
+ static public class Initial extends EvalFunc<Tuple>
+ {
+ int numSamples;
+ private Reservoir reservoir;
+ protected ScoredTuple.ScoreGenerator scoreGen;
+ TupleFactory tupleFactory = TupleFactory.getInstance();
+
+ public Initial(){}
+
+ public Initial(String numSamples)
+ {
+ this.numSamples = Integer.parseInt(numSamples);
+ }
+
+ private Reservoir getReservoir()
+ {
+ if (reservoir == null) {
+ reservoir = new Reservoir(this.numSamples);
+ }
+ return reservoir;
+ }
+
+ protected ScoredTuple.ScoreGenerator getScoreGenerator()
+ {
+ if(this.scoreGen == null)
+ {
+ this.scoreGen = new ScoredTuple.PureRandomScoreGenerator();
+ }
+ return this.scoreGen;
+ }
+
+ @Override
+ public Tuple exec(Tuple input) throws IOException {
+ DataBag output = BagFactory.getInstance().newDefaultBag();
+
+ ScoredTuple.ScoreGenerator scoreGen = getScoreGenerator();
+
+ DataBag samples = (DataBag) input.get(0);
+ if (samples == null)
+ {
+ // do nothing
+ }
+ else if (samples.size() <= numSamples) {
+ // no need to construct a reservoir, so just emit intermediate tuples
+ for (Tuple sample : samples) {
+ // add the score on to the intermediate tuple
+ output.add(new ScoredTuple(scoreGen.generateScore(sample), sample).getIntermediateTuple(tupleFactory));
+ }
+ } else {
+ getReservoir().clear();
+
+ for (Tuple sample : samples) {
+ getReservoir().consider(new ScoredTuple(scoreGen.generateScore(sample), sample));
+ }
+
+ for (ScoredTuple scoredTuple : getReservoir()) {
+ // add the score on to the intermediate tuple
+ output.add(scoredTuple.getIntermediateTuple(tupleFactory));
+ }
+ }
+
+ return tupleFactory.newTuple(output);
+ }
+
+ }
+
+ static public class Intermediate extends EvalFunc<Tuple>
+ {
+ int numSamples;
+ private Reservoir reservoir;
+ TupleFactory tupleFactory = TupleFactory.getInstance();
+
+ public Intermediate(){}
+
+ public Intermediate(String numSamples)
+ {
+ this.numSamples = Integer.parseInt(numSamples);
+ }
+
+ private Reservoir getReservoir()
+ {
+ if (reservoir == null) {
+ reservoir = new Reservoir(this.numSamples);
+ }
+ return reservoir;
+ }
+
+ @Override
+ public Tuple exec(Tuple input) throws IOException {
+ getReservoir().clear();
+
+ DataBag bagOfSamples = (DataBag) input.get(0);
+ for (Tuple innerTuple : bagOfSamples) {
+ DataBag samples = (DataBag) innerTuple.get(0);
+
+ for (Tuple sample : samples) {
+ // use the same score as previously generated
+ getReservoir().consider(ScoredTuple.fromIntermediateTuple(sample));
+ }
+ }
+
+ DataBag output = BagFactory.getInstance().newDefaultBag();
+ for (ScoredTuple scoredTuple : getReservoir()) {
+ // add the score on to the intermediate tuple
+ output.add(scoredTuple.getIntermediateTuple(tupleFactory));
+ }
+
+ return tupleFactory.newTuple(output);
+ }
+
+ }
+
+ static public class Final extends EvalFunc<DataBag>
+ {
+ int numSamples;
+ private Reservoir reservoir;
+ TupleFactory tupleFactory = TupleFactory.getInstance();
+
+ public Final(){}
+
+ public Final(String numSamples)
+ {
+ this.numSamples = Integer.parseInt(numSamples);
+ }
+
+ private Reservoir getReservoir()
+ {
+ if (reservoir == null) {
+ reservoir = new Reservoir(this.numSamples);
+ }
+ return reservoir;
+ }
+
+ @Override
+ public DataBag exec(Tuple input) throws IOException {
+ getReservoir().clear();
+
+ DataBag bagOfSamples = (DataBag) input.get(0);
+ for (Tuple innerTuple : bagOfSamples) {
+ DataBag samples = (DataBag) innerTuple.get(0);
+
+ for (Tuple sample : samples) {
+ // use the same score as previously generated
+ getReservoir().consider(ScoredTuple.fromIntermediateTuple(sample));
+ }
+ }
+
+ DataBag output = BagFactory.getInstance().newDefaultBag();
+ for (ScoredTuple scoredTuple : getReservoir()) {
+ // output the original tuple
+ output.add(scoredTuple.getTuple());
+ }
+
+ return output;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/sampling/SampleByKey.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/sampling/SampleByKey.java b/datafu-pig/src/main/java/datafu/pig/sampling/SampleByKey.java
new file mode 100644
index 0000000..90ea576
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/sampling/SampleByKey.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.sampling;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.security.MessageDigest;
+import org.apache.pig.FilterFunc;
+import org.apache.pig.data.Tuple;
+
+/**
+ * Provides a way of sampling tuples based on certain fields.
+ * This is essentially equivalent to grouping on the fields, applying SAMPLE,
+ * and then flattening. It is much more efficient though because it does not require
+ * a reduce step.
+ *
+ * <p>
+ * The method of sampling is to convert the key to a hash, derive a double value
+ * from this, and then test this against a supplied probability. The double value
+ * derived from a key is uniformly distributed between 0 and 1.
+ * </p>
+ *
+ * <p>
+ * The only required parameter is the sampling probability. This may be followed
+ * by an optional seed value to control the random number generation.
+ * </p>
+ *
+ * <p>
+ * SampleByKey will work deterministically as long as the same seed is provided.
+ * </p>
+ *
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * DEFINE SampleByKey datafu.pig.sampling.SampleByKey('0.5');
+ *
+ *-- input: (A,1), (A,2), (A,3), (B,1), (B,3)
+ *
+ * data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
+ * output = FILTER data BY SampleByKey(A_id);
+ *
+ * --output: (B,1), (B,3)
+ * }
+ *
+ * </pre>
+ * </p>
+ * @author evion
+ *
+ */
+
+public class SampleByKey extends FilterFunc
+{
+ final static int PRIME_NUMBER = 31;
+
+ Integer seed = null;
+ double probability;
+
+ public SampleByKey(String probability) {
+ this.probability = Double.parseDouble(probability);
+ }
+
+ public SampleByKey(String probability, String salt) {
+ this(probability);
+ this.seed = salt.hashCode();
+ }
+
+ @Override
+ public void setUDFContextSignature(String signature)
+ {
+ if (this.seed == null)
+ this.seed = signature.hashCode();
+ super.setUDFContextSignature(signature);
+ }
+
+ @Override
+ public Boolean exec(Tuple input) throws IOException
+ {
+ int hashCode = 0;
+ for(int i=0; i<input.size(); i++) {
+ Object each = input.get(i);
+ hashCode = hashCode*PRIME_NUMBER + each.hashCode();
+ }
+
+ try {
+ return intToRandomDouble(hashCode) <= probability;
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ throw new RuntimeException("Exception on intToRandomDouble");
+ }
+ }
+
+ private Double intToRandomDouble(int input) throws Exception
+ {
+ MessageDigest hasher = MessageDigest.getInstance("sha-1");
+
+ ByteBuffer b = ByteBuffer.allocate(4+4);
+ ByteBuffer b2 = ByteBuffer.allocate(20);
+
+ b.putInt(seed);
+ b.putInt(input);
+ byte[] digest = hasher.digest(b.array());
+ b.clear();
+
+ b2.put(digest);
+ b2.rewind();
+ double result = (((double)b2.getInt())/Integer.MAX_VALUE + 1)/2;
+ b2.clear();
+
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/sampling/ScoredTuple.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/sampling/ScoredTuple.java b/datafu-pig/src/main/java/datafu/pig/sampling/ScoredTuple.java
new file mode 100644
index 0000000..c793584
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/sampling/ScoredTuple.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.sampling;
+
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+
+class ScoredTuple implements Comparable<ScoredTuple>
+{
+ Double score;
+ private Tuple tuple;
+
+ public ScoredTuple()
+ {
+
+ }
+
+ public ScoredTuple(Double score, Tuple tuple)
+ {
+ this.score = score;
+ this.setTuple(tuple);
+ }
+
+ public Double getScore() {
+ return score;
+ }
+
+ public void setScore(Double score) {
+ this.score = score;
+ }
+
+ public Tuple getTuple() {
+ return tuple;
+ }
+
+ public void setTuple(Tuple tuple) {
+ this.tuple = tuple;
+ }
+
+ public Tuple getIntermediateTuple(TupleFactory tupleFactory)
+ {
+ Tuple intermediateTuple = tupleFactory.newTuple(2);
+ try {
+ intermediateTuple.set(0, score);
+ intermediateTuple.set(1, tuple);
+ }
+ catch (ExecException e) {
+ throw new RuntimeException(e);
+ }
+
+ return intermediateTuple;
+ }
+
+ public static ScoredTuple fromIntermediateTuple(Tuple intermediateTuple) throws ExecException
+ {
+ //Double score = ((Number)intermediateTuple.get(0)).doubleValue();
+ try {
+ Double score = (Double)intermediateTuple.get(0);
+ Tuple originalTuple = (Tuple)intermediateTuple.get(1);
+ return new ScoredTuple(score, originalTuple);
+ } catch (Exception e) {
+ throw new RuntimeException("Cannot deserialize intermediate tuple: "+intermediateTuple.toString(), e);
+ }
+ }
+
+ @Override
+ public int compareTo(ScoredTuple o) {
+ if (score == null) {
+ if (o == null) return 0;
+ else return -1;
+ }
+ return score.compareTo(o.score);
+ }
+
+ static interface ScoreGenerator
+ {
+ double generateScore(Tuple sample) throws ExecException;
+ }
+
+ static class PureRandomScoreGenerator implements ScoreGenerator
+ {
+ public PureRandomScoreGenerator(){}
+
+ public double generateScore(Tuple sample)
+ {
+ return Math.random();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/sampling/SimpleRandomSample.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/sampling/SimpleRandomSample.java b/datafu-pig/src/main/java/datafu/pig/sampling/SimpleRandomSample.java
new file mode 100644
index 0000000..8e8debf
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/sampling/SimpleRandomSample.java
@@ -0,0 +1,489 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.sampling;
+
+import java.io.IOException;
+import java.util.Comparator;
+
+import org.apache.commons.math.random.RandomDataImpl;
+import org.apache.pig.AlgebraicEvalFunc;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * Scalable simple random sampling (ScaSRS).
+ * <p/>
+ * This UDF implements a scalable simple random sampling algorithm described in
+ *
+ * <pre>
+ * X. Meng, Scalable Simple Random Sampling and Stratified Sampling, ICML 2013.
+ * </pre>
+ *
+ * It takes a bag of n items and a sampling probability p as the inputs, and outputs a
+ * simple random sample of size exactly ceil(p*n) in a bag, with probability at least
+ * 99.99%. For example, the following script generates a simple random sample with
+ * sampling probability 0.1:
+ *
+ * <pre>
+ * DEFINE SRS datafu.pig.sampling.SimpleRandomSample();
+ *
+ * item = LOAD 'input' AS (x:double);
+ * sampled = FOREACH (GROUP item ALL) GENERATE FLATTEN(SRS(item, 0.01));
+ * </pre>
+ *
+ * Optionally, user can provide a good lower bound of n as the third argument to help
+ * reduce the size of intermediate data, for example:
+ *
+ * <pre>
+ * DEFINE SRS datafu.pig.sampling.SimpleRandomSample();
+ *
+ * item = LOAD 'input' AS (x:double);
+ * summary = FOREACH (GROUP item ALL) GENERATE COUNT(item) AS count;
+ * sampled = FOREACH (GROUP item ALL) GENERATE FLATTEN(SRS(item, 0.01, summary.count));
+ * </pre>
+ *
+ * This UDF is very useful for stratified sampling. For example, the following script
+ * keeps all positive examples while downsampling negatives with probability 0.1:
+ *
+ * <pre>
+ * DEFINE SRS datafu.pig.sampling.SimpleRandomSample();
+ *
+ * item = LOAD 'input' AS (x:double, label:int);
+ * grouped = FOREACH (GROUP item BY label) GENERATE item, (group == 1 ? 1.0 : 0.1) AS p;
+ * sampled = FOREACH grouped GENERATE FLATTEN(SRS(item, p));
+ * </pre>
+ *
+ * In a Java Hadoop MapReduce job, we can output selected items directly using
+ * MultipleOutputs. However, this feature is not available in a Pig UDF. So we still let
+ * selected items go through the sort phase. However, as long as the sample size is not
+ * huge, this should not be a big problem.
+ *
+ * In the first version, the sampling probability is specified in the constructor. This
+ * method is deprecated now and will be removed in the next release.
+ *
+ * @author ximeng
+ *
+ */
+public class SimpleRandomSample extends AlgebraicEvalFunc<DataBag>
+{
+ /**
+ * Prefix for the output bag name.
+ */
+ public static final String OUTPUT_BAG_NAME_PREFIX = "SRS";
+
+ private static final TupleFactory _TUPLE_FACTORY = TupleFactory.getInstance();
+ private static final BagFactory _BAG_FACTORY = BagFactory.getInstance();
+
+ public SimpleRandomSample()
+ {
+ // empty
+ }
+
+ /**
+ * Constructs this UDF with a sampling probability.
+ *
+ * @deprecated Should specify the sampling probability in the function call.
+ */
+ @Deprecated
+ public SimpleRandomSample(String samplingProbability)
+ {
+ double p = Double.parseDouble(samplingProbability);
+ verifySamplingProbability(p);
+ }
+
+ @Override
+ public String getInitial()
+ {
+ return Initial.class.getName();
+ }
+
+ @Override
+ public String getIntermed()
+ {
+ return Intermediate.class.getName();
+ }
+
+ @Override
+ public String getFinal()
+ {
+ return Final.class.getName();
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try
+ {
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.BAG)
+ {
+ throw new RuntimeException("Expected a BAG as input");
+ }
+
+ return new Schema(new Schema.FieldSchema(super.getSchemaName(OUTPUT_BAG_NAME_PREFIX,
+ input),
+ inputFieldSchema.schema,
+ DataType.BAG));
+ }
+ catch (FrontendException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ static public class Initial extends EvalFunc<Tuple>
+ {
+ // Should avoid creating many random number generator instances.
+ private static RandomDataImpl _RNG = new RandomDataImpl();
+
+ synchronized private static double nextDouble()
+ {
+ return _RNG.nextUniform(0.0d, 1.0d);
+ }
+
+ public Initial()
+ {
+ // empty
+ }
+
+ @Deprecated
+ public Initial(String samplingProbability)
+ {
+ _p = Double.parseDouble(samplingProbability);
+ }
+
+ private boolean _first = true;
+ private double _p = -1.0d; // the sampling probability
+ private long _n1 = 0L; // the input lower bound of the size of the population
+ private long _localCount = 0L; // number of items processed by this instance
+
+ @Override
+ public Tuple exec(Tuple input) throws IOException
+ {
+ int numArgs = input.size();
+
+ // The first if clause is for backward compatibility, which should be removed
+ // after we remove specifying sampling probability in the constructor.
+ if(numArgs == 1)
+ {
+ if(_p < 0.0d)
+ {
+ throw new IllegalArgumentException("Sampling probability is not given.");
+ }
+ }
+ else if (numArgs < 2 || numArgs > 3)
+ {
+ throw new IllegalArgumentException("The input tuple should have either two or three fields: "
+ + "a bag of items, the sampling probability, "
+ + "and optionally a good lower bound of the size of the population or the exact number.");
+ }
+
+ DataBag items = (DataBag) input.get(0);
+ long numItems = items.size();
+ _localCount += numItems;
+
+ // This is also for backward compatibility. Should change to
+ // double p = ((Number) input.get(1)).doubleValue();
+ // after we remove specifying sampling probability in the constructor.
+ double p = numArgs == 1 ? _p : ((Number) input.get(1)).doubleValue();
+ if (_first)
+ {
+ _p = p;
+ verifySamplingProbability(p);
+ }
+ else
+ {
+ if (p != _p)
+ {
+ throw new IllegalArgumentException("The sampling probability must be a scalar, but found two different values: "
+ + _p + " and " + p + ".");
+ }
+ }
+
+ long n1 = 0L;
+ if (numArgs > 2)
+ {
+ n1 = ((Number) input.get(2)).longValue();
+
+ if (_first)
+ {
+ _n1 = n1;
+ }
+ else
+ {
+ if (n1 != _n1)
+ {
+ throw new IllegalArgumentException("The lower bound of the population size must be a scalar, but found two different values: "
+ + _n1 + " and " + n1 + ".");
+ }
+ }
+ }
+
+ _first = false;
+
+ // Use the local count if the input lower bound is smaller.
+ n1 = Math.max(n1, _localCount);
+
+ DataBag selected = _BAG_FACTORY.newDefaultBag();
+ DataBag waiting = _BAG_FACTORY.newDefaultBag();
+
+ if (n1 > 0L)
+ {
+ double q1 = getQ1(n1, p);
+ double q2 = getQ2(n1, p);
+
+ for (Tuple t : items)
+ {
+ double x = nextDouble();
+ if (x < q1)
+ {
+ selected.add(t);
+ }
+ else if (x < q2)
+ {
+ waiting.add(new ScoredTuple(x, t).getIntermediateTuple(_TUPLE_FACTORY));
+ }
+ }
+ }
+
+ /*
+ * The output tuple contains the following fields: sampling probability (double),
+ * number of processed items in this tuple (long), a good lower bound of the size of
+ * the population or the exact number (long), a bag of selected items (bag), and a
+ * bag of waitlisted items with scores (bag).
+ */
+ Tuple output = _TUPLE_FACTORY.newTuple();
+
+ output.append(p);
+ output.append(numItems);
+ output.append(n1);
+ output.append(selected);
+ output.append(waiting);
+
+ return output;
+ }
+ }
+
+ public static class Intermediate extends EvalFunc<Tuple>
+ {
+ public Intermediate()
+ {
+ // empty
+ }
+
+ @Deprecated
+ public Intermediate(String samplingProbability)
+ {
+ // empty
+ }
+
+ @Override
+ public Tuple exec(Tuple input) throws IOException
+ {
+ DataBag bag = (DataBag) input.get(0);
+
+ DataBag selected = _BAG_FACTORY.newDefaultBag();
+ DataBag aggWaiting = _BAG_FACTORY.newDefaultBag();
+
+ boolean first = true;
+ double p = 0.0d;
+ long numItems = 0L; // number of items processed, including rejected
+ long n1 = 0L;
+
+ for (Tuple tuple : bag)
+ {
+ if (first)
+ {
+ p = (Double) tuple.get(0);
+ first = false;
+ }
+
+ numItems += (Long) tuple.get(1);
+ n1 = Math.max((Long) tuple.get(2), numItems);
+
+ selected.addAll((DataBag) tuple.get(3));
+ aggWaiting.addAll((DataBag) tuple.get(4));
+ }
+
+ DataBag waiting = _BAG_FACTORY.newDefaultBag();
+
+ if (n1 > 0L)
+ {
+ double q1 = getQ1(n1, p);
+ double q2 = getQ2(n1, p);
+
+ for (Tuple t : aggWaiting)
+ {
+ ScoredTuple scored = ScoredTuple.fromIntermediateTuple(t);
+
+ if (scored.getScore() < q1)
+ {
+ selected.add(scored.getTuple());
+ }
+ else if (scored.getScore() < q2)
+ {
+ waiting.add(t);
+ }
+ }
+ }
+
+ Tuple output = _TUPLE_FACTORY.newTuple();
+
+ output.append(p);
+ output.append(numItems);
+ output.append(n1);
+ output.append(selected);
+ output.append(waiting);
+
+ return output;
+ }
+ }
+
+ static public class Final extends EvalFunc<DataBag>
+ {
+ public Final()
+ {
+ // empty
+ }
+
+ @Deprecated
+ public Final(String samplingProbability)
+ {
+ // empty
+ }
+
+ @Override
+ public DataBag exec(Tuple input) throws IOException
+ {
+ DataBag bag = (DataBag) input.get(0);
+
+ boolean first = true;
+ double p = 0.0d; // the sampling probability
+ long n = 0L; // the size of the population (total number of items)
+
+ DataBag selected = _BAG_FACTORY.newDefaultBag();
+ DataBag waiting = _BAG_FACTORY.newSortedBag(ScoredTupleComparator.getInstance());
+
+ for (Tuple tuple : bag)
+ {
+ if (first)
+ {
+ p = (Double) tuple.get(0);
+ first = false;
+ }
+
+ n += (Long) tuple.get(1);
+ selected.addAll((DataBag) tuple.get(3));
+ waiting.addAll((DataBag) tuple.get(4));
+ }
+
+ long numSelected = selected.size();
+ long numWaiting = waiting.size();
+
+ long s = (long) Math.ceil(p * n); // sample size
+
+ System.out.println("To sample " + s + " items from " + n + ", we pre-selected "
+ + numSelected + ", and waitlisted " + waiting.size() + ".");
+
+ long numNeeded = s - selected.size();
+
+ if (numNeeded < 0)
+ {
+ System.err.println("Pre-selected " + numSelected + " items, but only needed " + s
+ + ".");
+ }
+
+ for (Tuple scored : waiting)
+ {
+ if (numNeeded <= 0)
+ {
+ break;
+ }
+ selected.add(ScoredTuple.fromIntermediateTuple(scored).getTuple());
+ numNeeded--;
+ }
+
+ if (numNeeded > 0)
+ {
+ System.err.println("The waiting list only has " + numWaiting
+ + " items, but needed " + numNeeded + " more.");
+ }
+
+ return selected;
+ }
+ }
+
+ // computes a threshold to select items
+ private static double getQ1(long n, double p)
+ {
+ double t1 = 20.0d / (3.0d * n);
+ double q1 = p + t1 - Math.sqrt(t1 * t1 + 3.0d * t1 * p);
+ return q1;
+ }
+
+ // computes a threshold to reject items
+ private static double getQ2(long n, double p)
+ {
+ double t2 = 10.0d / n;
+ double q2 = p + t2 + Math.sqrt(t2 * t2 + 2.0d * t2 * p);
+ return q2;
+ }
+
+ private static void verifySamplingProbability(double p)
+ {
+ if(p < 0.0 || p > 1.0)
+ {
+ throw new IllegalArgumentException("Sampling probabiilty must be inside [0, 1].");
+ }
+ }
+
+ static class ScoredTupleComparator implements Comparator<Tuple>
+ {
+ public static final ScoredTupleComparator getInstance()
+ {
+ return _instance;
+ }
+
+ private static final ScoredTupleComparator _instance = new ScoredTupleComparator();
+
+ @Override
+ public int compare(Tuple o1, Tuple o2)
+ {
+ try
+ {
+ ScoredTuple t1 = ScoredTuple.fromIntermediateTuple(o1);
+ ScoredTuple t2 = ScoredTuple.fromIntermediateTuple(o2);
+ return t1.getScore().compareTo(t2.getScore());
+ }
+ catch (Throwable e)
+ {
+ throw new RuntimeException("Cannot compare " + o1 + " and " + o2 + ".", e);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/sampling/SimpleRandomSampleWithReplacementElect.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/sampling/SimpleRandomSampleWithReplacementElect.java b/datafu-pig/src/main/java/datafu/pig/sampling/SimpleRandomSampleWithReplacementElect.java
new file mode 100644
index 0000000..a59816a
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/sampling/SimpleRandomSampleWithReplacementElect.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.sampling;
+
+import java.io.IOException;
+import java.util.Comparator;
+
+import org.apache.pig.AlgebraicEvalFunc;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * Select the candidate with the smallest score for each position from the candidates
+ * proposed by {@link SimpleRandomSampleWithReplacementVote}.
+ *
+ * @see SimpleRandomSampleWithReplacementVote
+ *
+ * @author ximeng
+ *
+ */
+public class SimpleRandomSampleWithReplacementElect extends AlgebraicEvalFunc<DataBag>
+{
+ /**
+ * Prefix for the output bag name.
+ */
+ public static final String OUTPUT_BAG_NAME_PREFIX = "SRSWR_ELECT";
+
+ public static final TupleFactory tupleFactory = TupleFactory.getInstance();
+ public static final BagFactory bagFactory = BagFactory.getInstance();
+
+ static class CandidateComparator implements Comparator<Tuple>
+ {
+ private static CandidateComparator _instance = new CandidateComparator();
+
+ public static CandidateComparator get()
+ {
+ return _instance;
+ }
+
+ private CandidateComparator()
+ {
+ // singleton
+ }
+
+ @Override
+ public int compare(Tuple o1, Tuple o2)
+ {
+ try
+ {
+ // first by position
+ int c1 = ((Integer) o1.get(0)).compareTo((Integer) o2.get(0));
+ if (c1 != 0)
+ {
+ return c1;
+ }
+ else
+ {
+ // then by score
+ return ((Double) o1.get(1)).compareTo((Double) o2.get(1));
+ }
+ }
+ catch (ExecException e)
+ {
+ throw new RuntimeException("Error comparing tuples " + o1 + " and " + o2, e);
+ }
+ }
+ }
+
+ @Override
+ public String getInitial()
+ {
+ return Initial.class.getName();
+ }
+
+ @Override
+ public String getIntermed()
+ {
+ return Intermediate.class.getName();
+ }
+
+ @Override
+ public String getFinal()
+ {
+ return Final.class.getName();
+ }
+
+ static public class Initial extends EvalFunc<Tuple>
+ {
+ @Override
+ public Tuple exec(Tuple input) throws IOException
+ {
+ // output each input candidate
+ return input;
+ }
+ }
+
+ static public class Intermediate extends EvalFunc<Tuple>
+ {
+ @Override
+ public Tuple exec(Tuple tuple) throws IOException
+ {
+ // sort candidates first by index, then by key
+ DataBag candidates = bagFactory.newSortedBag(CandidateComparator.get());
+ for (Tuple intermediateOutputTuple : (DataBag) tuple.get(0))
+ {
+ candidates.addAll((DataBag) intermediateOutputTuple.get(0));
+ }
+
+ DataBag outputBag = bagFactory.newDefaultBag();
+ int i = -1;
+ for (Tuple candidate : candidates)
+ {
+ int pos = (Integer) candidate.get(0);
+ if (pos > i)
+ {
+ outputBag.add(candidate);
+ i = pos;
+ }
+ }
+
+ return tupleFactory.newTuple(outputBag);
+ }
+
+ }
+
+ static public class Final extends EvalFunc<DataBag>
+ {
+ @Override
+ public DataBag exec(Tuple tuple) throws IOException
+ {
+ DataBag candidates = bagFactory.newSortedBag(CandidateComparator.get());
+ for (Tuple intermediateOutputTuple : (DataBag) tuple.get(0))
+ {
+ candidates.addAll((DataBag) intermediateOutputTuple.get(0));
+ }
+
+ DataBag outputBag = bagFactory.newDefaultBag();
+ int i = -1;
+ for (Tuple candidate : candidates)
+ {
+ int pos = (Integer) candidate.get(0);
+ if (pos > i)
+ {
+ outputBag.add((Tuple) candidate.get(2));
+ i = pos;
+ }
+ }
+ return outputBag;
+ }
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try
+ {
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.BAG)
+ {
+ throw new RuntimeException("Expected a BAG as input");
+ }
+
+ // the output is a bag of selected items
+ Schema outputSchema =
+ new Schema(new Schema.FieldSchema(super.getSchemaName(OUTPUT_BAG_NAME_PREFIX,
+ input),
+ inputFieldSchema.schema.getField(0).schema.getField(2).schema,
+ DataType.BAG));
+
+ return outputSchema;
+ }
+ catch (FrontendException e)
+ {
+ throw new RuntimeException("Error deriving output schema.", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/sampling/SimpleRandomSampleWithReplacementVote.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/sampling/SimpleRandomSampleWithReplacementVote.java b/datafu-pig/src/main/java/datafu/pig/sampling/SimpleRandomSampleWithReplacementVote.java
new file mode 100644
index 0000000..598e58c
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/sampling/SimpleRandomSampleWithReplacementVote.java
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.sampling;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.math.MathException;
+import org.apache.commons.math.random.RandomDataImpl;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.primitives.Ints;
+
+/**
+ * Scalable simple random sampling with replacement (ScaSRSWR).
+ * <p/>
+ * This UDF together with {@link SimpleRandomSampleWithReplacementElect} implement a
+ * scalable algorithm for simple random sampling with replacement (SRSWR), which is a
+ * randomized algorithm with a failure rate less than {@value #FAILURE_RATE}.
+ * <p/>
+ * Let s be the desired sample size. To compute an SRSWR sample of size s, for each output
+ * position in {0, 1, ..., s-1}, we want to select an item from the population uniformly
+ * at random. This algorithm consists of two stages: vote and election. In the vote stage,
+ * this UDF {@link SimpleRandomSampleWithReplacementVote} votes items, called candidates,
+ * for each position. In the election stage, the paired UDF
+ * {@link SimpleRandomSampleWithReplacementElect} elects one candidate for each position.
+ * The algorithm succeeds if we have at least one candidate for each position.
+ * <p/>
+ * To use this UDF pair, user needs to provide: 1) the desired sample size, 2) a good
+ * lower bound of the population size or the exact size. The input to the vote UDF
+ * {@link SimpleRandomSampleWithReplacementVote} is a tuple that consists of a bag of
+ * items, the desired sample size (int), and the population size (long) or a good lower
+ * bound of it, where the latter two must be scalars. The output from the vote UDF is a
+ * tuple that consists of position:int, score:double, and candidate. The input to the
+ * elect UDF {@link SimpleRandomSampleWithReplacementElect} is a tuple that contains all
+ * candidates voted by the vote UDF for some positions. The output from the elect UDF is a
+ * bag of sampled items.
+ * <p/>
+ * For example, the following script generates a sample of size 100000 with replacement:
+ *
+ * <pre>
+ * DEFINE SRSWR_VOTE datafu.pig.sampling.SimpleRandomSampleWithReplacementVote();
+ * DEFINE SRSWR_ELECT datafu.pig.sampling.SimpleRandomSampleWithReplacementElect();
+ *
+ * item = LOAD 'input' AS (x:double);
+ * summary = FOREACH (GROUP item ALL) GENERATE COUNT(item) AS count;
+ * candidates = FOREACH item GENERATE FLATTEN(SRSWR_VOTE(TOBAG(x), 100000, summary.count));
+ * sampled = FOREACH (GROUP candidates BY position PARALLEL 10) GENERATE FLATTEN(SRSWR_ELECT(candidates));
+ * </pre>
+ *
+ * Because for election we only need to group candidates voted for the same position, this
+ * algorithm can use many reducers to consume the candidates. See the "PARALLEL 10"
+ * statement above. If the item to sample is the entire row, use TOBAG(TOTUPLE(*)).
+ * <p/>
+ * SRSWR is heavily used in bootstrapping. Bootstrapping can be done easily with this UDF
+ * pair. For example, the following script generates 100 bootstrap samples, computes the
+ * mean value for each sample, and then outputs the bootstrap estimates.
+ *
+ * <pre>
+ * summary = FOREACH (GROUP item ALL) GENERATE AVG(item.x) AS mean, COUNT(item) AS count;
+ * candidates = FOREACH item GENERATE FLATTEN(SRSWR_VOTE(TOBAG(x), summary.count*100, summary.count));
+ * sampled = FOREACH (GROUP candidates BY (position % 100) PARALLEL 10) GENERATE AVG(SRSWR_ELECT(candidates)) AS mean;
+ * bootstrap = FOREACH (GROUP sampled ALL) GENERATE summary.mean AS mean, sampled.mean AS bootstrapMeans;
+ * </pre>
+ *
+ * Another usage of this UDF pair is to generate random pairs or tuples without computing
+ * the cross product, where each pair or tuple consist of items from different input
+ * sources. Let s be the number of random tuples we want to generate. For each input
+ * source, simply use the vote UDF to propose candidates, then join the candidates from
+ * different sources by their positions and for each position use the elect UDF to select
+ * one candidate from each source to form the pair or tuple for that position.
+ * <p/>
+ * The algorithm is a simple extension to the work
+ *
+ * <pre>
+ * X. Meng, Scalable Simple Random Sampling and Stratified Sampling, ICML 2013.
+ * </pre>
+ *
+ * Basically, for each output position, it performs a random sort on the population
+ * (associates each item with a random score independently drawn from the uniform
+ * distribution and then sorts items based on the scores), and picks the one that has the
+ * smallest score. However, a probabilistic threshold is used to avoid sorting the entire
+ * population. For example, if the population size is one billion and the random score
+ * generated for an item is 0.9, very likely it won't become the smallest and hence we do
+ * not need to propose it as a candidate.
+ * <p/>
+ * More precisely, let n be the population size, n1 be a good lower bound of n, s be the
+ * sample size, delta be the failure rate, and q be the threshold. For each output
+ * position the probability of all random scores being greater than q is (1-q)^n. Thus, if
+ * we throw away items with associated scores greater than q, with probability at least 1
+ * - s*(1-q)^n, we can still capture the item with the smallest score for each position.
+ * Fix delta = s*(1-q)^n and solve for q, we get q = 1-exp(log(delta/s)/n), Note that
+ * replacing n by n1 < n can only decrease the failure rate, though at the cost of
+ * increased number of candidates. The expected number of candidates is (1 -
+ * exp(log(delta/s)/n1)*s*n. When n1 equals n, this number is approximately
+ * s*log(s/delta).
+ * <p/>
+ * Generating a random score for each (item, position) pair is very expensive and
+ * unnecessary. For each item, the number of positions for which it gets voted follows a
+ * binomial distribution B(s,q). We can simply draw a number from this distribution,
+ * determine the positions by sampling without replacement, and then generate random
+ * scores for those positions. This reduces the running time significantly.
+ * <p/>
+ * Since for each position we only need the candidate with the smallest score, we
+ * implement a combiner to reduce the size of intermediate data in the elect UDF
+ * {@link SimpleRandomSampleWithReplacementElect}.
+ *
+ * @see SimpleRandomSampleWithReplacementElect
+ * @see <a href="http://en.wikipedia.org/wiki/Bootstrapping_(statistics) target="_blank
+ * ">Boostrapping (Wikipedia)</a>
+ *
+ * @author ximeng
+ *
+ */
+public class SimpleRandomSampleWithReplacementVote extends EvalFunc<DataBag>
+{
+ public static final String OUTPUT_BAG_NAME_PREFIX = "SRSWR_VOTE";
+ public static final String CANDIDATE_FIELD_NAME = "candidate";
+ public static final String POSITION_FIELD_NAME = "position";
+ public static final String SCORE_FIELD_NAME = "score";
+ public static final double FAILURE_RATE = 1e-4;
+
+ private static final TupleFactory tupleFactory = TupleFactory.getInstance();
+ private static final BagFactory bagFactory = BagFactory.getInstance();
+
+ private RandomDataImpl _rdg = new RandomDataImpl();
+
+ /**
+ * Samples k integers from [0, n) without replacement efficiently.
+ *
+ * If k is small, we can repeatedly draw integers from [0, n) until there are k distinct
+ * values. For each trial, with probability at least (n-k)/n, we can draw a new value.
+ * So the expected number of trials is smaller than (k*n)/(n-k), which is a very rough
+ * bound. If k is large, we use the selection-rejection sampling algorithm. Basically,
+ * we want the running time to be O(k).
+ *
+ */
+ private int[] sampleWithoutReplacement(int n, int k)
+ {
+ if (k == 0)
+ {
+ return new int[] {};
+ }
+
+ if (k < n / 3L)
+ {
+ Set<Integer> sample = Sets.newHashSetWithExpectedSize(k);
+
+ // The expected number of iterations is less than 1.5*k
+ while (sample.size() < k)
+ {
+ sample.add(_rdg.nextInt(0, n - 1));
+ }
+
+ return Ints.toArray(sample);
+ }
+ else
+ {
+ int[] sample = new int[k];
+
+ int i = 0;
+ for (int j = 0; j < n && i < k; ++j)
+ {
+ if (_rdg.nextUniform(0.0d, 1.0d) < 1.0d * (k - i) / (n - j))
+ {
+ sample[i] = j;
+ i++;
+ }
+ }
+
+ return sample;
+ }
+ }
+
+ @Override
+ public DataBag exec(Tuple tuple) throws IOException
+ {
+ if (tuple.size() != 3)
+ {
+ throw new IllegalArgumentException("The input arguments are: "
+ + "a bag of items, the desired sample size (int), and the population size (long) or a good lower bound of it");
+ }
+
+ DataBag items = (DataBag) tuple.get(0);
+ int sampleSize = ((Number) tuple.get(1)).intValue();
+ long count = ((Number) tuple.get(2)).longValue();
+
+ /*
+ * The following threshold is to guarantee that each output position contains at least
+ * one candidate with high probability.
+ */
+ double threshold = 1.0d - Math.exp(Math.log(FAILURE_RATE / sampleSize) / count);
+
+ DataBag candidates = bagFactory.newDefaultBag();
+
+ for (Tuple item : items)
+ {
+ // Should be able to support long sample size if nextBinomial supports long.
+ int numOutputPositions;
+ try
+ {
+ numOutputPositions = _rdg.nextBinomial(sampleSize, threshold);
+ }
+ catch (MathException e)
+ {
+ throw new RuntimeException("Failed to generate a binomial value with n = "
+ + sampleSize + " and p = " + threshold, e);
+ }
+ for (int outputPosition : sampleWithoutReplacement(sampleSize, numOutputPositions))
+ {
+ Tuple candidate = tupleFactory.newTuple();
+ candidate.append(outputPosition);
+ candidate.append(_rdg.nextUniform(0.0d, 1.0d)); // generate a random score
+ candidate.append(item);
+ candidates.add(candidate);
+ }
+ }
+
+ return candidates;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try
+ {
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.BAG)
+ {
+ throw new RuntimeException("Expected a BAG as input");
+ }
+
+ List<Schema.FieldSchema> fieldSchemas = Lists.newArrayList();
+
+ fieldSchemas.add(new Schema.FieldSchema(POSITION_FIELD_NAME, DataType.INTEGER));
+ fieldSchemas.add(new Schema.FieldSchema(SCORE_FIELD_NAME, DataType.DOUBLE));
+ fieldSchemas.add(new Schema.FieldSchema(CANDIDATE_FIELD_NAME,
+ inputFieldSchema.schema.getField(0).schema));
+
+ Schema outputSchema =
+ new Schema(new Schema.FieldSchema(super.getSchemaName(OUTPUT_BAG_NAME_PREFIX,
+ input),
+ new Schema(fieldSchemas),
+ DataType.BAG));
+
+ return outputSchema;
+ }
+ catch (FrontendException e)
+ {
+ throw new RuntimeException("Error deriving output schema.", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/sampling/WeightedReservoirSample.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/sampling/WeightedReservoirSample.java b/datafu-pig/src/main/java/datafu/pig/sampling/WeightedReservoirSample.java
new file mode 100644
index 0000000..92af6a3
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/sampling/WeightedReservoirSample.java
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.sampling;
+
+import java.util.List;
+
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.builtin.Nondeterministic;
+import org.apache.pig.backend.executionengine.ExecException;
+
+/**
+ * <p>
+ * Performs a weighted random sample using an in-memory reservoir to produce
+ * a weighted random sample of a given size based on the A-Res algorithm described in
+ * {@link <a href="http://utopia.duth.gr/~pefraimi/research/data/2007EncOfAlg.pdf" target="_blank">paper</a>}.
+ * </p>
+ * <p>
+ * Species with larger weight have higher probability to be selected in the final sample set.
+ * </p>
+ * <p>
+ * This UDF inherits from {@link ReservoirSample} and it is guaranteed to produce
+ * a sample of the given size. Similarly it comes at the cost of scalability.
+ * since it uses internal storage with size equaling the desired sample to guarantee the exact sample size.
+ * </p>
+ * <p>
+ * Its constructor takes 2 arguments.
+ * <ul>
+ * <li>The 1st argument specifies the sample size which should be a string of positive integer.
+ * <li>The 2nd argument specifies the index of the weight field in the input tuple,
+ * which should be a string of non-negative integer that is no greater than the input tuple size.
+ * </ul>
+ * </p>
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * define WeightedSample datafu.pig.sampling.WeightedReservoirSample('1','1');
+ * input = LOAD 'input' AS (v1:chararray, v2:INT);
+ * input_g = GROUP input ALL;
+ * sampled = FOREACH input_g GENERATE WeightedSample(input);
+ * }
+ * </pre>
+ * </p>
+ * @author wjian
+ */
+
+@Nondeterministic
+public class WeightedReservoirSample extends ReservoirSample {
+
+ private Integer weightIdx;
+
+ public WeightedReservoirSample(String strNumSamples, String strWeightIdx)
+ {
+ super(strNumSamples);
+ this.weightIdx = Integer.parseInt(strWeightIdx);
+ if(this.weightIdx < 0) {
+ throw new IllegalArgumentException("Invalid negative index of weight field argument for WeightedReserviorSample constructor: "
+ + strWeightIdx);
+ }
+ }
+
+ @Override
+ protected ScoredTuple.ScoreGenerator getScoreGenerator()
+ {
+ if(super.scoreGen == null)
+ {
+ super.scoreGen = new InverseWeightScoreGenerator(this.weightIdx);
+ }
+ return this.scoreGen;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input) {
+ try {
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.BAG) {
+ throw new RuntimeException("Expected a BAG as input");
+ }
+
+ Schema inputBagSchema = inputFieldSchema.schema;
+
+ if (inputBagSchema.getField(0).type != DataType.TUPLE)
+ {
+ throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
+ DataType.findTypeName(inputBagSchema.getField(0).type)));
+ }
+
+ Schema tupleSchema = inputBagSchema.getField(0).schema;
+
+ if(tupleSchema == null) {
+ throw new RuntimeException("The tuple of input bag has no schema");
+ }
+
+ List<Schema.FieldSchema> fieldSchemaList = tupleSchema.getFields();
+
+ if(fieldSchemaList == null || fieldSchemaList.size() <= Math.max(0, this.weightIdx)) {
+ throw new RuntimeException("The field schema of the input tuple is null " +
+ "or the tuple size is no more than the weight field index: "
+ + this.weightIdx);
+ }
+
+ if(fieldSchemaList.get(this.weightIdx).type != DataType.INTEGER &&
+ fieldSchemaList.get(this.weightIdx).type != DataType.LONG &&
+ fieldSchemaList.get(this.weightIdx).type != DataType.FLOAT &&
+ fieldSchemaList.get(this.weightIdx).type != DataType.DOUBLE)
+ {
+ String[] expectedTypes = new String[] {DataType.findTypeName(DataType.INTEGER),
+ DataType.findTypeName(DataType.LONG),
+ DataType.findTypeName(DataType.FLOAT),
+ DataType.findTypeName(DataType.DOUBLE)};
+ throw new RuntimeException("Expect the type of the weight field of the input tuple to be of (" +
+ java.util.Arrays.toString(expectedTypes) + "), but instead found (" +
+ DataType.findTypeName(fieldSchemaList.get(this.weightIdx).type) + "), weight field: " +
+ this.weightIdx);
+ }
+
+ return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
+ inputFieldSchema.schema, DataType.BAG));
+ } catch (FrontendException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ }
+
+ String param = null;
+
+ private String getParam()
+ {
+ if (this.param == null) {
+ if(super.numSamples != null && this.weightIdx != null) {
+ this.param = String.format("('%d','%d')",
+ super.numSamples,
+ this.weightIdx);
+ } else {
+ this.param = "";
+ }
+ }
+
+ return this.param;
+ }
+
+
+ @Override
+ public String getInitial()
+ {
+ return Initial.class.getName() + getParam();
+ }
+
+ @Override
+ public String getIntermed()
+ {
+ return Intermediate.class.getName() + getParam();
+ }
+
+ @Override
+ public String getFinal()
+ {
+ return Final.class.getName() + getParam();
+ }
+
+ static public class Initial extends ReservoirSample.Initial
+ {
+ private Integer weightIdx;
+
+ public Initial()
+ {
+ super();
+ this.weightIdx = null;
+ }
+
+ public Initial(String strNumSamples, String strWeightIdx)
+ {
+ super(strNumSamples);
+ this.weightIdx = Integer.parseInt(strWeightIdx);
+ if(this.weightIdx < 0) {
+ throw new IllegalArgumentException("Invalid negative index of weight field for WeightedReserviorSample.Initial constructor: "
+ + strWeightIdx);
+ }
+ }
+
+ @Override
+ protected ScoredTuple.ScoreGenerator getScoreGenerator()
+ {
+ if(super.scoreGen == null)
+ {
+ super.scoreGen = new InverseWeightScoreGenerator(this.weightIdx);
+ }
+ return super.scoreGen;
+ }
+ }
+
+ static public class Intermediate extends ReservoirSample.Intermediate
+ {
+ public Intermediate()
+ {
+ super();
+ }
+
+ public Intermediate(String strNumSamples, String strWeightIdx)
+ {
+ super(strNumSamples);
+ }
+ }
+
+ static public class Final extends ReservoirSample.Final
+ {
+ public Final()
+ {
+ super();
+ }
+
+ public Final(String strNumSamples, String strWeightIdx)
+ {
+ super(strNumSamples);
+ }
+ }
+
+ static class InverseWeightScoreGenerator implements ScoredTuple.ScoreGenerator
+ {
+ //index of the weight field of the input tuple
+ private int weightIdx;
+
+ InverseWeightScoreGenerator(Integer weightIdx)
+ {
+ if(weightIdx == null || weightIdx < 0) {
+ throw new IllegalArgumentException("Invalid null or negative weight index input: " + weightIdx);
+ }
+ this.weightIdx = weightIdx;
+ }
+
+ @Override
+ public double generateScore(Tuple sample) throws ExecException
+ {
+ if(this.weightIdx >= sample.size())
+ {
+ throw new ExecException(String.format("Weight index %d is outside tuple bounds", this.weightIdx));
+ }
+ if (sample.get(this.weightIdx) == null)
+ {
+ throw new ExecException(String.format("null value for weight at index %d",this.weightIdx));
+ }
+ double weight = ((Number)sample.get(this.weightIdx)).doubleValue();
+ if(Double.compare(weight, 0.0) <= 0)
+ {
+ //non-positive weight should be avoided
+ throw new ExecException(String.format("Invalid sample weight [%f]. It should be a positive real number", weight));
+ }
+ //a differnt approach to try: u^(1/w) could be exp(log(u)/w) ?
+ return Math.pow(Math.random(), 1/weight);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/sampling/WeightedSample.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/sampling/WeightedSample.java b/datafu-pig/src/main/java/datafu/pig/sampling/WeightedSample.java
new file mode 100644
index 0000000..a77164d
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/sampling/WeightedSample.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.sampling;
+
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.pig.EvalFunc;
+import org.apache.pig.builtin.Nondeterministic;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * Performs weighted bernoulli sampling on a bag.
+ *
+ * <p>
+ * Create a new bag by performing a weighted sampling without replacement
+ * from the input bag. Sampling is biased according to a weight that
+ * is part of the inner tuples in the bag. That is, tuples with relatively
+ * high weights are more likely to be chosen over tuples with low weights.
+ * Optionally, a limit on the number of items to return may be specified.
+ * </p>
+ *
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * define WeightedSample datafu.pig.sampling.WeightedSample()
+ *
+ * -- input:
+ * -- ({(a, 100),(b, 1),(c, 5),(d, 2)})
+ * input = LOAD 'input' AS (A: bag{T: tuple(name:chararray,score:int)});
+ *
+ * output1 = FOREACH input GENERATE WeightedSample(A,1);
+ * -- output1:
+ * -- uses the field indexed by 1 as a score
+ * -- ({(a,100),(c,5),(b,1),(d,2)}) -- example of random
+ *
+ * -- sample using the second column (index 1) and keep only the top 3
+ * output2 = FOREACH input GENERATE WeightedSample(A,1,3);
+ * -- output2:
+ * -- ({(a,100),(c,5),(b,1)})
+ * }
+ * </pre>
+ */
+@Nondeterministic
+public class WeightedSample extends EvalFunc<DataBag>
+{
+ BagFactory bagFactory = BagFactory.getInstance();
+ Long seed = null;
+
+ public WeightedSample() {
+ }
+
+ public WeightedSample(String seed) {
+ this.seed = Long.parseLong(seed);
+ }
+
+ @Override
+ public DataBag exec(Tuple input) throws IOException {
+ DataBag output = bagFactory.newDefaultBag();
+
+ DataBag samples = (DataBag) input.get(0);
+ if (samples == null || samples.size() == 0) {
+ return output; // if we are given null we will return an empty bag
+ }
+ int numSamples = (int) samples.size();
+ if (numSamples == 1) return samples;
+
+ Tuple[] tuples = new Tuple[numSamples];
+ int tupleIndex = 0;
+ for (Tuple tuple : samples) {
+ tuples[tupleIndex] = tuple;
+ tupleIndex++;
+ }
+
+ double[] scores = new double[numSamples];
+ int scoreIndex = ((Number)input.get(1)).intValue();
+ tupleIndex = 0;
+ for (Tuple tuple : samples) {
+ double score = ((Number)tuple.get(scoreIndex)).doubleValue();
+ score = Math.max(score, Double.MIN_NORMAL); // negative scores cause problems
+ scores[tupleIndex] = score;
+ tupleIndex++;
+ }
+
+ // accept any type of number for sample size, but convert to int
+ int limitSamples = numSamples;
+ if (input.size() == 3) {
+ // sample limit included
+ limitSamples = Math.min(((Number)input.get(2)).intValue(), numSamples);
+ }
+
+ /*
+ * Here's how the algorithm works:
+ *
+ * 1. Create a cumulative distribution of the scores 2. Draw a random number 3. Find
+ * the interval in which the drawn number falls into 4. Select the element
+ * encompassing that interval 5. Remove the selected element from consideration 6.
+ * Repeat 1-5 k times
+ *
+ * However, rather than removing the element (#5), which is expensive for an array,
+ * this function performs some extra bookkeeping by replacing the selected element
+ * with an element from the front of the array and truncating the front. This
+ * complicates matters as the element positions have changed, so another mapping for
+ * positions is needed.
+ *
+ * This is an O(k*n) algorithm, where k is the number of elements to sample and n is
+ * the number of scores.
+ */
+ Random rng = null;
+ if (seed == null) {
+ rng = new Random();
+ } else {
+ rng = new Random(seed);
+ }
+
+ for (int k = 0; k < limitSamples; k++) {
+ double val = rng.nextDouble();
+ int idx = find_cumsum_interval(scores, val, k, numSamples);
+ if (idx == numSamples)
+ idx = rng.nextInt(numSamples - k) + k;
+
+ output.add(tuples[idx]);
+
+ scores[idx] = scores[k];
+ tuples[idx] = tuples[k];
+ }
+
+ return output;
+ }
+
+ public int find_cumsum_interval(double[] scores, double val, int begin, int end) {
+ double sum = 0.0;
+ double cumsum = 0.0;
+ for (int i = begin; i < end; i++) {
+ sum += scores[i];
+ }
+
+ for (int i = begin; i < end; i++) {
+ cumsum += scores[i];
+ if ((cumsum / sum) > val)
+ return i;
+ }
+ return end;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input) {
+ try {
+ if (!(input.size() == 2 || input.size() == 3))
+ {
+ throw new RuntimeException("Expected input to have two or three fields");
+ }
+
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.BAG) {
+ throw new RuntimeException("Expected a BAG as first input, got: "+inputFieldSchema.type);
+ }
+
+ if (input.getField(1).type != DataType.INTEGER) {
+ throw new RuntimeException("Expected an INT as second input, got: "+input.getField(1).type);
+ }
+
+ if (input.size() == 3 && !(input.getField(2).type == DataType.INTEGER || input.getField(2).type == DataType.LONG)) {
+ throw new RuntimeException("Expected an INT or LONG as second input, got: "+input.getField(2).type);
+ }
+
+ return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
+ inputFieldSchema.schema, DataType.BAG));
+ } catch (FrontendException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/sampling/package-info.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/sampling/package-info.java b/datafu-pig/src/main/java/datafu/pig/sampling/package-info.java
new file mode 100644
index 0000000..0c7d8bd
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/sampling/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Sampling UDFs, including weighted sample, reservoir sampling, sampling by key, etc.
+ */
+package datafu.pig.sampling;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/sessions/SessionCount.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/sessions/SessionCount.java b/datafu-pig/src/main/java/datafu/pig/sessions/SessionCount.java
new file mode 100644
index 0000000..fe68888
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/sessions/SessionCount.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.sessions;
+
+import java.io.IOException;
+
+import org.apache.pig.Accumulator;
+import org.apache.pig.AccumulatorEvalFunc;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.joda.time.DateTime;
+import org.joda.time.Period;
+
+import datafu.pig.util.SimpleEvalFunc;
+
+/**
+ * Performs a count of events, ignoring events which occur within the
+ * same time window.
+ * <p>
+ * This is useful for tasks such as counting the number of page views per user since it:
+ * a) prevent reloads and go-backs from overcounting actual views
+ * b) captures the notion that views across multiple sessions are more meaningful
+ * <p>
+ * Input <b>must</b> be sorted ascendingly by time for this UDF to work.
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ *
+ * %declare TIME_WINDOW 10m
+ *
+ * define SessionCount datafu.pig.sessions.SessionCount('$TIME_WINDOW');
+ *
+ * views = LOAD 'views' as (user_id:int, page_id:int, time:chararray);
+ * views_grouped = GROUP views by (user_id, page_id);
+ * view_counts = FOREACH views_grouped {
+ * views = order views by time;
+ * generate group.user_id as user_id,
+ * group.page_id as page_id,
+ * SessionCount(views.(time)) as count; }
+ * }
+ * </pre>
+ *
+ */
+public class SessionCount extends AccumulatorEvalFunc<Long>
+{
+ private final long millis;
+ private DateTime last_date;
+ private long sum;
+
+ public SessionCount(String timeSpec)
+ {
+ Period p = new Period("PT" + timeSpec.toUpperCase());
+ this.millis = p.toStandardSeconds().getSeconds() * 1000;
+ cleanup();
+ }
+
+ @Override
+ public void accumulate(Tuple input) throws IOException
+ {
+ for (Tuple t : (DataBag) input.get(0)) {
+ DateTime date = new DateTime(t.get(0));
+
+ if (last_date == null) {
+ last_date = date;
+ sum = 1;
+ } else if (date.isAfter(last_date.plus(this.millis)))
+ sum += 1;
+ else if (date.isBefore(last_date))
+ throw new IOException("input time series is not sorted");
+
+ last_date = date;
+ }
+ }
+
+ @Override
+ public Long getValue()
+ {
+ return sum;
+ }
+
+ @Override
+ public void cleanup()
+ {
+ this.last_date = null;
+ this.sum = 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/sessions/Sessionize.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/sessions/Sessionize.java b/datafu-pig/src/main/java/datafu/pig/sessions/Sessionize.java
new file mode 100644
index 0000000..52d159b
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/sessions/Sessionize.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.sessions;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.pig.Accumulator;
+import org.apache.pig.AccumulatorEvalFunc;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.builtin.Nondeterministic;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.joda.time.DateTime;
+import org.joda.time.Period;
+
+/**
+ * Sessionizes an input stream, appending a session ID to each tuple.
+ *
+ * <p>
+ * This UDF takes a constructor argument which is the session timeout (an idle
+ * period of this amount indicates that a new session has started) and assumes
+ * the first element of the input bag is an ISO8601 timestamp. The input bag
+ * must be sorted by this timestamp. It returns the input bag with a new field,
+ * session_id, that is a GUID indicating the session of the request.
+ * </p>
+ *
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ *
+ * %declare TIME_WINDOW 30m
+ *
+ * define Sessionize datafu.pig.sessions.Sessionize('$TIME_WINDOW');
+ *
+ * views = LOAD 'views.tsv' AS (visit_date:chararray, member_id:int, url:chararray);
+ *
+ * -- sessionize the visit stream
+ * views = GROUP views BY member_id;
+ * sessions = FOREACH views {
+ * visits = ORDER views BY visit_date;
+ * GENERATE FLATTEN(Sessionize(VISITS)) AS (visit_date,member_id,url,session_id);
+ * }
+ *
+ * -- count the number of sessions hitting the url
+ * rollup = GROUP sessions BY url;
+ * result = FOREACH rollup GENERATE group AS url, COUNT(SESSIONS) AS session_cnt;
+ * }
+ * </pre>
+ * </p>
+ */
+@Nondeterministic
+public class Sessionize extends AccumulatorEvalFunc<DataBag>
+{
+ private final long millis;
+
+ private DataBag outputBag;
+ private DateTime last_date;
+ private String id;
+
+ public Sessionize(String timeSpec)
+ {
+ Period p = new Period("PT" + timeSpec.toUpperCase());
+ this.millis = p.toStandardSeconds().getSeconds() * 1000;
+
+ cleanup();
+ }
+
+ @Override
+ public void accumulate(Tuple input) throws IOException
+ {
+ for (Tuple t : (DataBag) input.get(0)) {
+ Object timeObj = t.get(0);
+
+ DateTime date;
+ if (timeObj instanceof String)
+ {
+ date = new DateTime((String)timeObj);
+ }
+ else if (timeObj instanceof Long)
+ {
+ date = new DateTime((Long)timeObj);
+ }
+ else
+ {
+ throw new RuntimeException("Time must either be a String or Long");
+ }
+
+ if (this.last_date == null)
+ this.last_date = date;
+ else if (date.isAfter(this.last_date.plus(this.millis)))
+ this.id = UUID.randomUUID().toString();
+ else if (date.isBefore(last_date))
+ throw new IOException(String.format("input time series is not sorted (%s < %s)", date, last_date));
+
+ Tuple t_new = TupleFactory.getInstance().newTuple(t.getAll());
+ t_new.append(this.id);
+ outputBag.add(t_new);
+
+ this.last_date = date;
+ }
+ }
+
+ @Override
+ public DataBag getValue()
+ {
+ return outputBag;
+ }
+
+ @Override
+ public void cleanup()
+ {
+ this.last_date = null;
+ this.outputBag = BagFactory.getInstance().newDefaultBag();
+ this.id = UUID.randomUUID().toString();
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try {
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.BAG)
+ {
+ throw new RuntimeException("Expected a BAG as input");
+ }
+
+ Schema inputBagSchema = inputFieldSchema.schema;
+
+ if (inputBagSchema.getField(0).type != DataType.TUPLE)
+ {
+ throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
+ DataType.findTypeName(inputBagSchema.getField(0).type)));
+ }
+
+ Schema inputTupleSchema = inputBagSchema.getField(0).schema;
+
+ if (inputTupleSchema.getField(0).type != DataType.CHARARRAY
+ && inputTupleSchema.getField(0).type != DataType.LONG)
+ {
+ throw new RuntimeException(String.format("Expected first element of tuple to be a CHARARRAY or LONG, but instead found %s",
+ DataType.findTypeName(inputTupleSchema.getField(0).type)));
+ }
+
+ Schema outputTupleSchema = inputTupleSchema.clone();
+ outputTupleSchema.add(new Schema.FieldSchema("session_id", DataType.CHARARRAY));
+
+ return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
+ .getName()
+ .toLowerCase(), input),
+ outputTupleSchema,
+ DataType.BAG));
+ }
+ catch (CloneNotSupportedException e) {
+ throw new RuntimeException(e);
+ }
+ catch (FrontendException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/sessions/package-info.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/sessions/package-info.java b/datafu-pig/src/main/java/datafu/pig/sessions/package-info.java
new file mode 100644
index 0000000..686fbf0
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/sessions/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * UDFs for sessionizing data.
+ */
+package datafu.pig.sessions;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/sets/SetDifference.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/sets/SetDifference.java b/datafu-pig/src/main/java/datafu/pig/sets/SetDifference.java
new file mode 100644
index 0000000..b6469e9
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/sets/SetDifference.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.sets;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.PriorityQueue;
+
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+
+/**
+ * Computes the set difference of two or more bags. Duplicates are eliminated. <b>The input bags must be sorted.</b>
+ *
+ * <p>
+ * If bags A and B are provided, then this computes A-B, i.e. all elements in A that are not in B.
+ * If bags A, B and C are provided, then this computes A-B-C, i.e. all elements in A that are not in B or C.
+ * </p>
+ *
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * define SetDifference datafu.pig.sets.SetDifference();
+ *
+ * -- input:
+ * -- ({(1),(2),(3),(4),(5),(6)},{(3),(4)})
+ * input = LOAD 'input' AS (B1:bag{T:tuple(val:int)},B2:bag{T:tuple(val:int)});
+ *
+ * input = FOREACH input {
+ * B1 = ORDER B1 BY val ASC;
+ * B2 = ORDER B2 BY val ASC;
+ *
+ * -- output:
+ * -- ({(1),(2),(5),(6)})
+ * GENERATE SetDifference(B1,B2);
+ * }
+ * }</pre>
+ */
+public class SetDifference extends SetOperationsBase
+{
+ private static final BagFactory bagFactory = BagFactory.getInstance();
+
+ /**
+ * Loads the data bags from the input tuple and puts them in a priority queue,
+ * where ordering is determined by the data from the iterator for each bag.
+ *
+ * <p>
+ * The bags are wrapped in a {@link Pair} object that is comparable on the data
+ * currently available from the iterator.
+ * These objects are ordered first by the data, then by the index within the tuple
+ * the bag came from.
+ * </p>
+ *
+ * @param input
+ * @return
+ * @throws IOException
+ */
+ private PriorityQueue<Pair> loadBags(Tuple input) throws IOException
+ {
+ PriorityQueue<Pair> pq = new PriorityQueue<Pair>(input.size());
+
+ for (int i=0; i < input.size(); i++)
+ {
+ if (input.get(i) != null)
+ {
+ Iterator<Tuple> inputIterator = ((DataBag)input.get(i)).iterator();
+ if(inputIterator.hasNext())
+ {
+ pq.add(new Pair(inputIterator,i));
+ }
+ }
+ }
+ return pq;
+ }
+
+ /**
+ * Counts how many elements in the priority queue match the
+ * element at the front of the queue, which should be from the first bag.
+ *
+ * @param pq priority queue
+ * @return number of matches
+ */
+ public int countMatches(PriorityQueue<Pair> pq)
+ {
+ Pair nextPair = pq.peek();
+ Tuple data = nextPair.data;
+
+ // sanity check
+ if (!nextPair.index.equals(0))
+ {
+ throw new RuntimeException("Expected next bag to have index 0");
+ }
+
+ int matches = 0;
+ for (Pair p : pq) {
+ if (data.equals(p.data))
+ matches++;
+ }
+ // subtract 1 since element matches itself
+ return matches - 1;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public DataBag exec(Tuple input) throws IOException
+ {
+ if (input.size() < 2)
+ {
+ throw new RuntimeException("Expected at least two inputs, but found " + input.size());
+ }
+
+ for (Object o : input)
+ {
+ if (o != null && !(o instanceof DataBag))
+ {
+ throw new RuntimeException("Inputs must be bags");
+ }
+ }
+
+ DataBag outputBag = bagFactory.newDefaultBag();
+
+ DataBag bag1 = (DataBag)input.get(0);
+ DataBag bag2 = (DataBag)input.get(1);
+
+ if (bag1 == null || bag1.size() == 0)
+ {
+ return outputBag;
+ }
+ // optimization
+ else if (input.size() == 2 && (bag2 == null || bag2.size() == 0))
+ {
+ return bag1;
+ }
+
+ PriorityQueue<Pair> pq = loadBags(input);
+
+ Tuple lastData = null;
+
+ while (true)
+ {
+ Pair nextPair = pq.peek();
+
+ // ignore data we've already encountered
+ if (nextPair.data.compareTo(lastData) != 0)
+ {
+ // Only take data from the first bag, where there are no other
+ // bags that have the same data.
+ if (nextPair.index.equals(0) && countMatches(pq) == 0)
+ {
+ outputBag.add(nextPair.data);
+ lastData = nextPair.data;
+ }
+ }
+
+ Pair p = pq.poll();
+
+ // only put the bag back into the queue if it still has data
+ if (p.hasNext())
+ {
+ p.next();
+ pq.offer(p);
+ }
+ else if (p.index.equals(0))
+ {
+ // stop when we exhaust all elements from the first bag
+ break;
+ }
+ }
+
+ return outputBag;
+ }
+
+ /**
+ * A wrapper for the tuple iterator that implements comparable so it can be used in the priority queue.
+ *
+ * <p>
+ * This is compared first on the data, then on the index the bag came from
+ * in the input tuple.
+ * </p>
+ * @author mhayes
+ *
+ */
+ private static class Pair implements Comparable<Pair>
+ {
+ private final Iterator<Tuple> it;
+ private final Integer index;
+ private Tuple data;
+
+ /**
+ * Constructs the {@link Pair}.
+ *
+ * @param it tuple iterator
+ * @param index index within the tuple that the bag came from
+ */
+ public Pair(Iterator<Tuple> it, int index)
+ {
+ this.index = index;
+ this.it = it;
+ this.data = it.next();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public int compareTo(Pair o)
+ {
+ int r = this.data.compareTo(o.data);
+ if (r == 0)
+ {
+ return index.compareTo(o.index);
+ }
+ else
+ {
+ return r;
+ }
+ }
+
+ public boolean hasNext()
+ {
+ return it.hasNext();
+ }
+
+ @SuppressWarnings("unchecked")
+ public Tuple next()
+ {
+ Tuple nextData = it.next();
+ // algorithm assumes data is in order
+ if (data.compareTo(nextData) > 0)
+ {
+ throw new RuntimeException("Out of order!");
+ }
+ this.data = nextData;
+ return this.data;
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("[%s within %d]",data,index);
+ }
+ }
+}
+
+
[13/19] DATAFU-27 Migrate build system to Gradle
Posted by wv...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/bags/BagTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/bags/BagTests.java b/datafu-pig/src/test/java/datafu/test/pig/bags/BagTests.java
new file mode 100644
index 0000000..f02f0cb
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/bags/BagTests.java
@@ -0,0 +1,1252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.bags;
+
+import static org.testng.Assert.assertEquals;
+
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+
+import junit.framework.Assert;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.pig.bags.CountEach;
+import datafu.pig.bags.DistinctBy;
+import datafu.pig.bags.Enumerate;
+import datafu.test.pig.PigTests;
+
+
+public class BagTests extends PigTests
+{
+ /**
+
+
+ define NullToEmptyBag datafu.pig.bags.NullToEmptyBag();
+
+ data = LOAD 'input' AS (B: bag {T: tuple(v:INT)});
+
+ dump data;
+
+ data2 = FOREACH data GENERATE NullToEmptyBag(B) as P;
+
+ dump data2;
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline
+ private String nullToEmptyBag;
+
+ @Test
+ public void nullToEmptyBagTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(nullToEmptyBag);
+
+ writeLinesToFile("input",
+ "({(1),(2),(3),(4),(5)})",
+ "()",
+ "{(4),(5)})");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "({(1),(2),(3),(4),(5)})",
+ "({})",
+ "({(4),(5)})");
+ }
+
+ /**
+
+
+ define EmptyBagToNull datafu.pig.bags.EmptyBagToNull();
+
+ data = LOAD 'input' AS (B: bag {T: tuple(v:INT)});
+
+ dump data;
+
+ data2 = FOREACH data GENERATE EmptyBagToNull(B) as P;
+
+ dump data2;
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline
+ private String emptyBagToNullTest;
+
+ @Test
+ public void emptyBagToNullTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(emptyBagToNullTest);
+
+ writeLinesToFile("input",
+ "({(1),(2),(3),(4),(5)})",
+ "()",
+ "({})",
+ "{(4),(5)})");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "({(1),(2),(3),(4),(5)})",
+ "()",
+ "()",
+ "({(4),(5)})");
+ }
+
+ /**
+
+
+ define EmptyBagToNullFields datafu.pig.bags.EmptyBagToNullFields();
+
+ data = LOAD 'input' AS (B: bag {T: tuple(v1:INT,v2:INT)});
+
+ dump data;
+
+ data2 = FOREACH data GENERATE EmptyBagToNullFields(B) as P;
+
+ dump data2;
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline
+ private String emptyBagToNullFieldsTest;
+
+ @Test
+ public void emptyBagToNullFieldsTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(emptyBagToNullFieldsTest);
+
+ writeLinesToFile("input",
+ "({(1,1),(2,2),(3,3),(4,4),(5,5)})",
+ "({})",
+ "{(4,4),(5,5)})");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "({(1,1),(2,2),(3,3),(4,4),(5,5)})",
+ "({(,)})",
+ "({(4,4),(5,5)})");
+ }
+
+ /**
+
+
+ define AppendToBag datafu.pig.bags.AppendToBag();
+
+ data = LOAD 'input' AS (key:INT, B: bag{T: tuple(v:INT)}, T: tuple(v:INT));
+
+ data2 = FOREACH data GENERATE key, AppendToBag(B,T) as B;
+
+ STORE data2 INTO 'output';
+
+ */
+ @Multiline
+ private String appendToBagTest;
+
+ @Test
+ public void appendToBagTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(appendToBagTest);
+
+ writeLinesToFile("input",
+ "1\t{(1),(2),(3)}\t(4)",
+ "2\t{(10),(20),(30),(40),(50)}\t(60)");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "(1,{(1),(2),(3),(4)})",
+ "(2,{(10),(20),(30),(40),(50),(60)})");
+ }
+
+ /**
+
+
+ define FirstTupleFromBag datafu.pig.bags.FirstTupleFromBag();
+
+ data = LOAD 'input' AS (key:INT, B: bag{T: tuple(v:INT)});
+
+ data2 = FOREACH data GENERATE key, FirstTupleFromBag(B, null) as B;
+
+ STORE data2 INTO 'output';
+
+ */
+ @Multiline
+ private String firstTupleFromBagTest;
+
+ @Test
+ public void firstTupleFromBagTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(firstTupleFromBagTest);
+
+ writeLinesToFile("input", "1\t{(4),(9),(16)}");
+
+ test.runScript();
+
+ assertOutput(test, "data2", "(1,(4))");
+ }
+
+ /**
+
+
+ define PrependToBag datafu.pig.bags.PrependToBag();
+
+ data = LOAD 'input' AS (key:INT, B: bag{T: tuple(v:INT)}, T: tuple(v:INT));
+
+ data2 = FOREACH data GENERATE key, PrependToBag(B,T) as B;
+
+ STORE data2 INTO 'output';
+
+ */
+ @Multiline
+ private String prependToBagTest;
+
+ @Test
+ public void prependToBagTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(prependToBagTest);
+
+ writeLinesToFile("input",
+ "1\t{(1),(2),(3)}\t(4)",
+ "2\t{(10),(20),(30),(40),(50)}\t(60)");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "(1,{(4),(1),(2),(3)})",
+ "(2,{(60),(10),(20),(30),(40),(50)})");
+ }
+
+ /**
+
+
+ define BagConcat datafu.pig.bags.BagConcat();
+
+ data = LOAD 'input' AS (A: bag{T: tuple(v:INT)}, B: bag{T: tuple(v:INT)}, C: bag{T: tuple(v:INT)});
+
+ describe data;
+
+ data2 = FOREACH data GENERATE BagConcat(A,B,C);
+
+ describe data2;
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline
+ private String bagConcatTest;
+
+ @Test
+ public void bagConcatTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(bagConcatTest);
+
+ writeLinesToFile("input",
+ "({(1),(2),(3)}\t{(3),(5),(6)}\t{(10),(13)})",
+ "({(2),(3),(4)}\t{(5),(5)}\t{(20)})");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "({(1),(2),(3),(3),(5),(6),(10),(13)})",
+ "({(2),(3),(4),(5),(5),(20)})");
+ }
+
+ /**
+
+
+ define UnorderedPairs datafu.pig.bags.UnorderedPairs();
+
+ data = LOAD 'input' AS (B: bag {T: tuple(v:INT)});
+
+ data2 = FOREACH data GENERATE UnorderedPairs(B) as P;
+
+ data3 = FOREACH data2 GENERATE FLATTEN(P);
+
+ data4 = FOREACH data3 GENERATE FLATTEN(elem1), FLATTEN(elem2);
+
+ data5 = ORDER data4 BY $0, $1;
+
+ STORE data5 INTO 'output';
+
+
+ */
+ @Multiline
+ private String unorderedPairsTest;
+
+ @Test
+ public void unorderedPairsTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(unorderedPairsTest);
+
+ String[] input = {
+ "{(1),(2),(3),(4),(5)}"
+ };
+
+ String[] output = {
+ "(1,2)",
+ "(1,3)",
+ "(1,4)",
+ "(1,5)",
+ "(2,3)",
+ "(2,4)",
+ "(2,5)",
+ "(3,4)",
+ "(3,5)",
+ "(4,5)"
+ };
+
+ test.assertOutput("data",input,"data4",output);
+ }
+
+ /**
+
+
+ define UnorderedPairs datafu.pig.bags.UnorderedPairs();
+
+ data = LOAD 'input' AS (A:int, B: bag {T: tuple(v:INT)});
+
+ data2 = FOREACH data GENERATE A, UnorderedPairs(B) as P;
+
+ data3 = FOREACH data2 GENERATE A, FLATTEN(P);
+
+ STORE data3 INTO 'output';
+
+ */
+ @Multiline
+ private String unorderedPairsTest2;
+
+ @Test
+ public void unorderedPairsTest2() throws Exception
+ {
+ PigTest test = createPigTestFromString(unorderedPairsTest2);
+
+ this.writeLinesToFile("input", "1\t{(1),(2),(3),(4),(5)}");
+
+ test.runScript();
+ this.getLinesForAlias(test, "data3");
+
+ this.assertOutput(test, "data3",
+ "(1,(1),(2))",
+ "(1,(1),(3))",
+ "(1,(1),(4))",
+ "(1,(1),(5))",
+ "(1,(2),(3))",
+ "(1,(2),(4))",
+ "(1,(2),(5))",
+ "(1,(3),(4))",
+ "(1,(3),(5))",
+ "(1,(4),(5))");
+ }
+
+ /**
+
+
+ define BagSplit datafu.pig.bags.BagSplit();
+
+ data = LOAD 'input' AS (B:bag{T:tuple(val1:INT,val2:INT)});
+
+ data2 = FOREACH data GENERATE BagSplit($MAX,B);
+ --describe data2;
+
+ data3 = FOREACH data2 GENERATE FLATTEN($0);
+
+ --describe data3
+
+ STORE data3 INTO 'output';
+
+ */
+ @Multiline
+ private String bagSplitTest;
+
+ @Test
+ public void bagSplitTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(bagSplitTest,
+ "MAX=5");
+
+ writeLinesToFile("input",
+ "{(1,11),(2,22),(3,33),(4,44),(5,55),(6,66),(7,77),(8,88),(9,99),(10,1010),(11,1111),(12,1212)}");
+
+ test.runScript();
+
+ assertOutput(test, "data3",
+ "({(1,11),(2,22),(3,33),(4,44),(5,55)})",
+ "({(6,66),(7,77),(8,88),(9,99),(10,1010)})",
+ "({(11,1111),(12,1212)})");
+ }
+
+ /**
+
+
+ define BagSplit datafu.pig.bags.BagSplit('true');
+
+ data = LOAD 'input' AS (B:bag{T:tuple(val1:INT,val2:INT)});
+
+ data2 = FOREACH data GENERATE BagSplit($MAX,B);
+
+ data3 = FOREACH data2 GENERATE FLATTEN($0);
+
+ STORE data3 INTO 'output';
+ */
+ @Multiline
+ private String bagSplitWithBagNumTest;
+
+ @Test
+ public void bagSplitWithBagNumTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(bagSplitWithBagNumTest,
+ "MAX=10");
+
+ writeLinesToFile("input",
+ "{(1,11),(2,22),(3,33),(4,44),(5,55),(6,66),(7,77),(8,88),(9,99),(10,1010),(11,1111),(12,1212)}");
+
+ test.runScript();
+
+ assertOutput(test, "data3",
+ "({(1,11),(2,22),(3,33),(4,44),(5,55),(6,66),(7,77),(8,88),(9,99),(10,1010)},0)",
+ "({(11,1111),(12,1212)},1)");
+ }
+
+ /**
+
+
+ define Enumerate datafu.pig.bags.ReverseEnumerate('1');
+
+ data = LOAD 'input' AS (data: bag {T: tuple(v1:INT,B: bag{T: tuple(v2:INT)})});
+
+ data2 = FOREACH data GENERATE Enumerate(data);
+ --describe data2;
+
+ data3 = FOREACH data2 GENERATE FLATTEN($0);
+ --describe data3;
+
+ data4 = FOREACH data3 GENERATE $0 as v1, $1 as B, $2 as i;
+ --describe data4;
+
+ STORE data4 INTO 'output';
+
+ */
+ @Multiline
+ private String enumerateWithReverseTest;
+
+ @Test
+ public void enumerateWithReverseTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(enumerateWithReverseTest);
+
+ writeLinesToFile("input",
+ "({(10,{(1),(2),(3)}),(20,{(4),(5),(6)}),(30,{(7),(8)}),(40,{(9),(10),(11)}),(50,{(12),(13),(14),(15)})})");
+
+ test.runScript();
+
+ assertOutput(test, "data4",
+ "(10,{(1),(2),(3)},5)",
+ "(20,{(4),(5),(6)},4)",
+ "(30,{(7),(8)},3)",
+ "(40,{(9),(10),(11)},2)",
+ "(50,{(12),(13),(14),(15)},1)");
+ }
+
+ /**
+
+
+ define Enumerate datafu.pig.bags.Enumerate('1');
+
+ data = LOAD 'input' AS (data: bag {T: tuple(v1:INT,B: bag{T: tuple(v2:INT)})});
+
+ data2 = FOREACH data GENERATE Enumerate(data);
+ --describe data2;
+
+ data3 = FOREACH data2 GENERATE FLATTEN($0);
+ --describe data3;
+
+ data4 = FOREACH data3 GENERATE $0 as v1, $1 as B, $2 as i;
+ --describe data4;
+
+ STORE data4 INTO 'output';
+
+ */
+ @Multiline
+ private String enumerateWithStartTest;
+
+ @Test
+ public void enumerateWithStartTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(enumerateWithStartTest);
+
+ writeLinesToFile("input",
+ "({(10,{(1),(2),(3)}),(20,{(4),(5),(6)}),(30,{(7),(8)}),(40,{(9),(10),(11)}),(50,{(12),(13),(14),(15)})})");
+
+ test.runScript();
+
+ assertOutput(test, "data4",
+ "(10,{(1),(2),(3)},1)",
+ "(20,{(4),(5),(6)},2)",
+ "(30,{(7),(8)},3)",
+ "(40,{(9),(10),(11)},4)",
+ "(50,{(12),(13),(14),(15)},5)");
+ }
+
+ /**
+
+
+ define Enumerate datafu.pig.bags.Enumerate();
+
+ data = LOAD 'input' AS (data: bag {T: tuple(v1:INT,B: bag{T: tuple(v2:INT)})});
+
+ data2 = FOREACH data GENERATE Enumerate(data);
+ --describe data2;
+
+ data3 = FOREACH data2 GENERATE FLATTEN($0);
+ --describe data3;
+
+ data4 = FOREACH data3 GENERATE $0 as v1, $1 as B, $2 as i;
+ --describe data4;
+
+ STORE data4 INTO 'output';
+
+ */
+ @Multiline
+ private String enumerateTest;
+
+ @Test
+ public void enumerateTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(enumerateTest);
+
+ writeLinesToFile("input",
+ "({(10,{(1),(2),(3)}),(20,{(4),(5),(6)}),(30,{(7),(8)}),(40,{(9),(10),(11)}),(50,{(12),(13),(14),(15)})})");
+
+ test.runScript();
+
+ assertOutput(test, "data4",
+ "(10,{(1),(2),(3)},0)",
+ "(20,{(4),(5),(6)},1)",
+ "(30,{(7),(8)},2)",
+ "(40,{(9),(10),(11)},3)",
+ "(50,{(12),(13),(14),(15)},4)");
+ }
+
+ @Test
+ public void enumerateTest2() throws Exception
+ {
+ PigTest test = createPigTestFromString(enumerateTest);
+
+ writeLinesToFile("input",
+ "({(10,{(1),(2),(3)}),(20,{(4),(5),(6)}),(30,{(7),(8)}),(40,{(9),(10),(11)}),(50,{(12),(13),(14),(15)})})",
+ "({(11,{(11),(12),(13),(14)}),(21,{(15),(16),(17),(18)}),(31,{(19),(20)}),(41,{(21),(22),(23),(24)}),(51,{(25),(26),(27)})})");
+
+ test.runScript();
+
+ assertOutput(test, "data4",
+ "(10,{(1),(2),(3)},0)",
+ "(20,{(4),(5),(6)},1)",
+ "(30,{(7),(8)},2)",
+ "(40,{(9),(10),(11)},3)",
+ "(50,{(12),(13),(14),(15)},4)",
+ "(11,{(11),(12),(13),(14)},0)",
+ "(21,{(15),(16),(17),(18)},1)",
+ "(31,{(19),(20)},2)",
+ "(41,{(21),(22),(23),(24)},3)",
+ "(51,{(25),(26),(27)},4)");
+ }
+
+ /*
+ * Testing "Accumulator" part of Enumeration by manually invoking accumulate(), getValue() and cleanup()
+ */
+ @Test
+ public void enumerateAccumulatorTest() throws Exception
+ {
+ Enumerate enumerate = new Enumerate();
+
+ Tuple tuple1 = TupleFactory.getInstance().newTuple(1);
+ tuple1.set(0, 10);
+
+ Tuple tuple2 = TupleFactory.getInstance().newTuple(1);
+ tuple2.set(0, 20);
+
+ Tuple tuple3 = TupleFactory.getInstance().newTuple(1);
+ tuple3.set(0, 30);
+
+ Tuple tuple4 = TupleFactory.getInstance().newTuple(1);
+ tuple4.set(0, 40);
+
+ Tuple tuple5 = TupleFactory.getInstance().newTuple(1);
+ tuple5.set(0, 50);
+
+ DataBag bag1 = BagFactory.getInstance().newDefaultBag();
+ bag1.add(tuple1);
+ bag1.add(tuple2);
+ bag1.add(tuple3);
+
+ DataBag bag2 = BagFactory.getInstance().newDefaultBag();
+ bag2.add(tuple4);
+ bag2.add(tuple5);
+
+ Tuple inputTuple1 = TupleFactory.getInstance().newTuple(1);
+ inputTuple1.set(0,bag1);
+
+ Tuple inputTuple2 = TupleFactory.getInstance().newTuple(1);
+ inputTuple2.set(0,bag2);
+
+ enumerate.accumulate(inputTuple1);
+ enumerate.accumulate(inputTuple2);
+ assertEquals(enumerate.getValue().toString(), "{(10,0),(20,1),(30,2),(40,3),(50,4)}");
+
+ // Testing that cleanup code is correct by calling cleanup() and passing inputs back to Enumerate instance
+ enumerate.cleanup();
+ enumerate.accumulate(inputTuple1);
+ enumerate.accumulate(inputTuple2);
+ assertEquals(enumerate.getValue().toString(), "{(10,0),(20,1),(30,2),(40,3),(50,4)}");
+ }
+
+ /**
+
+
+ define BagSplit datafu.pig.bags.BagSplit();
+ define Enumerate datafu.pig.bags.Enumerate('1');
+
+ data = LOAD 'input' AS (data: bag {T: tuple(name:CHARARRAY, score:double)});
+
+ data2 = FOREACH data GENERATE BagSplit(3,data) as the_bags;
+
+ --describe data2
+
+ data3 = FOREACH data2 GENERATE Enumerate(the_bags) as enumerated_bags;
+
+ --describe data3
+
+ data4 = FOREACH data3 GENERATE FLATTEN(enumerated_bags) as (data,i);
+
+ --describe data4
+
+ data5 = FOREACH data4 GENERATE data as the_data, i as the_key;
+
+ --describe data5
+
+ data_out = FOREACH data5 GENERATE FLATTEN(the_data), the_key;
+
+ --describe data_out
+ */
+ @Multiline
+ private String comprehensiveBagSplitAndEnumerate;
+
+ @Test
+ public void comprehensiveBagSplitAndEnumerate() throws Exception
+ {
+ PigTest test = createPigTestFromString(comprehensiveBagSplitAndEnumerate);
+
+ writeLinesToFile("input",
+ "({(A,1.0),(B,2.0),(C,3.0),(D,4.0),(E,5.0)})");
+
+ test.runScript();
+
+ assertOutput(test, "data_out",
+ // bag #1
+ "(A,1.0,1)",
+ "(B,2.0,1)",
+ "(C,3.0,1)",
+ // bag #2
+ "(D,4.0,2)",
+ "(E,5.0,2)");
+ }
+
+ /**
+
+
+ define DistinctBy datafu.pig.bags.DistinctBy('0');
+
+ data = LOAD 'input' AS (data: bag {T: tuple(a:CHARARRAY, b:INT, c:INT)});
+
+ data2 = FOREACH data GENERATE DistinctBy(data);
+
+ --describe data2;
+
+ STORE data2 INTO 'output';
+
+ */
+ @Multiline
+ private String distinctByTest;
+
+ @Test
+ public void distinctByTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(distinctByTest);
+
+ writeLinesToFile("input",
+ "({(Z,1,0),(A,1,0),(A,1,0),(B,2,0),(B,22,1),(C,3,0),(D,4,0),(E,5,0)})",
+ "({(A,10,2),(M,50,3),(A,34,49), (A,24,42), (Z,49,22),(B,1,1)},(B,2,2))");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "({(Z,1,0),(A,1,0),(B,2,0),(C,3,0),(D,4,0),(E,5,0)})",
+ "({(A,10,2),(M,50,3),(Z,49,22),(B,1,1)})");
+ }
+
+ /**
+
+ define DistinctBy datafu.pig.bags.DistinctBy('1', '2');
+
+ data = LOAD 'input' AS (data: bag {T: tuple(a:CHARARRAY, b:map[INT], c:bag{t: tuple(c0:CHARARRAY, c1:INT)})});
+
+ data2 = FOREACH data GENERATE DistinctBy(data);
+
+ --describe data2;
+
+ STORE data2 INTO 'output';
+
+ */
+ @Multiline
+ private String distinctByMultiComplexFieldTest;
+
+ @Test
+ public void distinctByMultiComplexFieldTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(distinctByMultiComplexFieldTest);
+
+ writeLinesToFile("input",
+ "({(a-b,[a#0,b#1],{(a-b,0),(a-b,1)}),(a-c,[b#1,a#0],{(a-b,0),(a-b,1)}),(a-d,[a#1,b#0],{(a-b,1),(a-b,2)})})");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "({(a-b,[b#1,a#0],{(a-b,0),(a-b,1)}),(a-d,[b#0,a#1],{(a-b,1),(a-b,2)})})");
+ }
+
+ /**
+
+ define DistinctBy datafu.pig.bags.DistinctBy('1');
+
+ data = LOAD 'input' AS (data: bag {T: tuple(a:CHARARRAY, b:CHARARRAY)});
+
+ data2 = FOREACH data GENERATE DistinctBy(data);
+
+ --describe data2;
+
+ STORE data2 INTO 'output';
+
+ */
+ @Multiline
+ private String distinctByDelimTest;
+
+ @Test
+ public void distinctByDelimTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(distinctByDelimTest);
+
+ writeLinesToFile("input",
+ "({(a-b,c),(a-b,d)})");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "({(a-b,c),(a-b,d)})");
+ }
+
+ @Test
+ public void distinctByExecTest() throws Exception
+ {
+ DistinctBy distinct = new DistinctBy("0");
+
+ DataBag bag;
+ Tuple input;
+ Tuple data;
+
+ bag = BagFactory.getInstance().newDefaultBag();
+ input = TupleFactory.getInstance().newTuple(bag);
+
+ data = TupleFactory.getInstance().newTuple(2);
+ bag.add(data);
+ data.set(0, 10);
+ data.set(1, 20);
+
+ data = TupleFactory.getInstance().newTuple(2);
+ bag.add(data);
+ data.set(0, 11);
+ data.set(1, 50);
+
+ data = TupleFactory.getInstance().newTuple(2);
+ bag.add(data);
+ data.set(0, 10);
+ data.set(1, 22);
+
+ data = TupleFactory.getInstance().newTuple(2);
+ bag.add(data);
+ data.set(0, 12);
+ data.set(1, 40);
+
+ data = TupleFactory.getInstance().newTuple(2);
+ bag.add(data);
+ data.set(0, 11);
+ data.set(1, 50);
+
+ data = TupleFactory.getInstance().newTuple(2);
+ bag.add(data);
+ data.set(0, 11);
+ data.set(1, 51);
+
+ DataBag result = distinct.exec(input);
+
+ Assert.assertEquals(3, result.size());
+
+ Iterator<Tuple> iter = result.iterator();
+ Assert.assertEquals("(10,20)", iter.next().toString());
+ Assert.assertEquals("(11,50)", iter.next().toString());
+ Assert.assertEquals("(12,40)", iter.next().toString());
+
+ // do it again to test cleanup
+ bag = BagFactory.getInstance().newDefaultBag();
+ input = TupleFactory.getInstance().newTuple(bag);
+
+ data = TupleFactory.getInstance().newTuple(2);
+ bag.add(data);
+ data.set(0, 12);
+ data.set(1, 42);
+
+ data = TupleFactory.getInstance().newTuple(2);
+ bag.add(data);
+ data.set(0, 11);
+ data.set(1, 51);
+
+ data = TupleFactory.getInstance().newTuple(2);
+ bag.add(data);
+ data.set(0, 11);
+ data.set(1, 50);
+
+ result = distinct.exec(input);
+
+ Assert.assertEquals(2, result.size());
+
+ iter = result.iterator();
+ Assert.assertEquals("(12,42)", iter.next().toString());
+ Assert.assertEquals("(11,51)", iter.next().toString());
+ }
+
+ @Test
+ public void distinctByAccumulateTest() throws Exception
+ {
+ DistinctBy distinct = new DistinctBy("0");
+
+ DataBag bag;
+ Tuple input;
+ Tuple data;
+
+ data = TupleFactory.getInstance().newTuple(2);
+ bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(data);
+ input = TupleFactory.getInstance().newTuple(bag);
+ data.set(0, 10);
+ data.set(1, 20);
+ distinct.accumulate(input);
+
+ data = TupleFactory.getInstance().newTuple(2);
+ bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(data);
+ input = TupleFactory.getInstance().newTuple(bag);
+ data.set(0, 11);
+ data.set(1, 50);
+ distinct.accumulate(input);
+
+ data = TupleFactory.getInstance().newTuple(2);
+ bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(data);
+ input = TupleFactory.getInstance().newTuple(bag);
+ data.set(0, 10);
+ data.set(1, 22);
+ distinct.accumulate(input);
+
+ data = TupleFactory.getInstance().newTuple(2);
+ bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(data);
+ input = TupleFactory.getInstance().newTuple(bag);
+ data.set(0, 12);
+ data.set(1, 40);
+ distinct.accumulate(input);
+
+ data = TupleFactory.getInstance().newTuple(2);
+ bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(data);
+ input = TupleFactory.getInstance().newTuple(bag);
+ data.set(0, 11);
+ data.set(1, 50);
+ distinct.accumulate(input);
+
+ data = TupleFactory.getInstance().newTuple(2);
+ bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(data);
+ input = TupleFactory.getInstance().newTuple(bag);
+ data.set(0, 11);
+ data.set(1, 51);
+ distinct.accumulate(input);
+
+ DataBag result = distinct.getValue();
+
+ Assert.assertEquals(3, result.size());
+
+ Iterator<Tuple> iter = result.iterator();
+ Assert.assertEquals("(10,20)", iter.next().toString());
+ Assert.assertEquals("(11,50)", iter.next().toString());
+ Assert.assertEquals("(12,40)", iter.next().toString());
+
+ // do it again to test cleanup
+ distinct.cleanup();
+
+ data = TupleFactory.getInstance().newTuple(2);
+ bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(data);
+ input = TupleFactory.getInstance().newTuple(bag);
+ data.set(0, 12);
+ data.set(1, 42);
+ distinct.accumulate(input);
+
+ data = TupleFactory.getInstance().newTuple(2);
+ bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(data);
+ input = TupleFactory.getInstance().newTuple(bag);
+ data.set(0, 11);
+ data.set(1, 51);
+ distinct.accumulate(input);
+
+ data = TupleFactory.getInstance().newTuple(2);
+ bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(data);
+ input = TupleFactory.getInstance().newTuple(bag);
+ data.set(0, 11);
+ data.set(1, 50);
+ distinct.accumulate(input);
+
+ result = distinct.getValue();
+
+ Assert.assertEquals(2, result.size());
+
+ iter = result.iterator();
+ Assert.assertEquals("(12,42)", iter.next().toString());
+ Assert.assertEquals("(11,51)", iter.next().toString());
+ }
+
+ /**
+
+
+ define CountEach datafu.pig.bags.CountEach();
+
+ data = LOAD 'input' AS (data: bag {T: tuple(v1:chararray)});
+
+ data2 = FOREACH data GENERATE CountEach(data) as counted;
+ --describe data2;
+
+ data3 = FOREACH data2 {
+ ordered = ORDER counted BY count DESC;
+ GENERATE ordered;
+ }
+ --describe data3
+
+ STORE data3 INTO 'output';
+
+ */
+ @Multiline
+ private String countEachTest;
+
+ @Test
+ public void countEachTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(countEachTest);
+
+ writeLinesToFile("input",
+ "({(A),(B),(A),(C),(A),(B)})");
+
+ test.runScript();
+
+ assertOutput(test, "data3",
+ "({((A),3),((B),2),((C),1)})");
+ }
+
+ @Test
+ public void countEachExecAndAccumulateTest() throws Exception
+ {
+ for (int c=0; c<2; c++)
+ {
+ CountEach countEach = new CountEach("flatten");
+
+ DataBag bag = BagFactory.getInstance().newDefaultBag();
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, "A");
+ bag.add(t);
+ }
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, "B");
+ bag.add(t);
+ }
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, "B");
+ bag.add(t);
+ }
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, "C");
+ bag.add(t);
+ }
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, "A");
+ bag.add(t);
+ }
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, "D");
+ bag.add(t);
+ }
+
+ DataBag output = null;
+
+ if (c == 0)
+ {
+ Tuple input = TupleFactory.getInstance().newTuple(1);
+ input.set(0, bag);
+
+ System.out.println("Testing exec");
+ output = countEach.exec(input);
+ }
+ else
+ {
+ System.out.println("Testing accumulate");
+ for (Tuple t : bag)
+ {
+ DataBag tb = BagFactory.getInstance().newDefaultBag();
+ tb.add(t);
+ Tuple input = TupleFactory.getInstance().newTuple(1);
+ input.set(0, tb);
+ countEach.accumulate(input);
+ }
+
+ output = countEach.getValue();
+
+ countEach.cleanup();
+ Assert.assertEquals(0, countEach.getValue().size());
+ }
+
+ System.out.println(output.toString());
+
+ Assert.assertEquals(4, output.size());
+ Set<String> found = new HashSet<String>();
+ for (Tuple t : output)
+ {
+ String key = (String)t.get(0);
+ found.add(key);
+ if (key == "A")
+ {
+ Assert.assertEquals(2, t.get(1));
+ }
+ else if (key == "B")
+ {
+ Assert.assertEquals(2, t.get(1));
+ }
+ else if (key == "C")
+ {
+ Assert.assertEquals(1, t.get(1));
+ }
+ else if (key == "D")
+ {
+ Assert.assertEquals(1, t.get(1));
+ }
+ else
+ {
+ Assert.fail("Unexpected: " + key);
+ }
+ }
+ Assert.assertEquals(4, found.size());
+ }
+ }
+
+ /**
+
+
+ define CountEach datafu.pig.bags.CountEach('flatten');
+
+ data = LOAD 'input' AS (data: bag {T: tuple(v1:chararray)});
+
+ data2 = FOREACH data GENERATE CountEach(data) as counted;
+ --describe data2;
+
+ data3 = FOREACH data2 {
+ ordered = ORDER counted BY count DESC;
+ GENERATE ordered;
+ }
+ --describe data3
+
+ STORE data3 INTO 'output';
+
+ */
+ @Multiline
+ private String countEachFlattenTest;
+
+ @Test
+ public void countEachFlattenTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(countEachFlattenTest);
+
+ writeLinesToFile("input",
+ "({(A),(B),(A),(C),(A),(B)})");
+
+ test.runScript();
+
+ assertOutput(test, "data3",
+ "({(A,3),(B,2),(C,1)})");
+ }
+
+ /**
+
+
+ define BagLeftOuterJoin datafu.pig.bags.BagLeftOuterJoin();
+
+ data = LOAD 'input' AS (outer_key:chararray, bag1:bag{T:tuple(k:chararray,v:chararray)}, bag2:bag{T:tuple(k:chararray,v:chararray)}, bag3:bag{T:tuple(k3:chararray,v3:chararray)});
+ describe data;
+
+ data2 = FOREACH data GENERATE
+ outer_key,
+ BagLeftOuterJoin(bag1, 'k', bag2, 'k', bag3, 'k3') as joined1,
+ BagLeftOuterJoin(bag1, 'k', bag3, 'k3', bag2, 'k') as joined2; --this will break without UDF signature and pig < 0.11
+ describe data2;
+
+ STORE data2 INTO 'output';
+
+ */
+ @Multiline
+ private String bagLeftOuterJoinTest;
+
+ @Test
+ public void bagLeftOuterJoinTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(bagLeftOuterJoinTest);
+
+ writeLinesToFile("input",
+ "1\t{(K1,A1),(K2,B1),(K3,C1)}\t{(K1,A2),(K2,B2),(K2,B22)}\t{(K1,A3),(K3,C3),(K4,D3)}");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "(1,{(K1,A1,K1,A2,K1,A3),(K2,B1,K2,B2,,),(K2,B1,K2,B22,,),(K3,C1,,,K3,C3)},{(K1,A1,K1,A3,K1,A2),(K2,B1,,,K2,B2),(K2,B1,,,K2,B22),(K3,C1,K3,C3,,)})");
+ }
+
+ /**
+
+
+ define BagUnion datafu.pig.bags.BagConcat();
+
+ data = LOAD 'input' AS (input_bag: bag {T: tuple(inner_bag: bag {T2: tuple(k: int, v: chararray)})});
+ describe data;
+
+ data2 = FOREACH data GENERATE BagUnion(input_bag) as unioned;
+ describe data2;
+
+ STORE data INTO 'output';
+
+ */
+ @Multiline
+ private String bagUnionTest;
+
+ @Test
+ public void bagUnionTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(bagUnionTest);
+ writeLinesToFile("input", "({({(1,A),(1,B)}),({(2,A),(2,B),(2,C)}),({(3,A)})}");
+ test.runScript();
+ assertOutput(test, "data2", "({(1,A),(1,B),(2,A),(2,B),(2,C),(3,A)})");
+ }
+
+ /**
+
+
+ define BagGroup datafu.pig.bags.BagGroup();
+
+ data = LOAD 'input' AS (input_bag: bag {T: tuple(k: int, v: chararray)});
+ describe data;
+
+ data2 = FOREACH data GENERATE BagGroup(input_bag, input_bag.k) as grouped;
+ describe data2;
+
+ data3 = FOREACH data2 {
+ ordered = ORDER grouped BY group;
+ GENERATE
+ ordered as grouped;
+ }
+ describe data3;
+
+ STORE data INTO 'output';
+
+ */
+ @Multiline
+ private String bagGroupSingleTest;
+
+ @Test
+ public void bagGroupSingleTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(bagGroupSingleTest);
+ writeLinesToFile("input", "({(1,A),(1,B),(2,A),(2,B),(2,C),(3,A)})");
+ test.runScript();
+ getLinesForAlias(test, "data2", true);
+ assertOutput(test, "data3", "({(1,{(1,A),(1,B)}),(2,{(2,A),(2,B),(2,C)}),(3,{(3,A)})})");
+ }
+
+ /**
+
+
+ define BagGroup datafu.pig.bags.BagGroup();
+
+ data = LOAD 'input' AS (input_bag: bag {T: tuple(k: int, k2: chararray, v: int)});
+ describe data;
+
+ data2 = FOREACH data GENERATE BagGroup(input_bag, input_bag.(k, k2)) as grouped;
+ describe data2;
+
+ data3 = FOREACH data2 {
+ ordered = ORDER grouped BY group;
+ GENERATE
+ ordered as grouped;
+ }
+ describe data3;
+
+ STORE data INTO 'output';
+
+ */
+ @Multiline
+ private String bagGroupMultipleTest;
+
+ @Test
+ public void bagGroupMultipleTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(bagGroupMultipleTest);
+ writeLinesToFile("input", "({(1,A,1),(1,B,1),(1,A,2),(2,A,1),(2,B,1),(2,C,1),(3,A,1)})");
+ test.runScript();
+ getLinesForAlias(test, "data2", true);
+ assertOutput(test, "data3", "({((1,A),{(1,A,1),(1,A,2)}),((1,B),{(1,B,1)}),((2,A),{(2,A,1)}),((2,B),{(2,B,1)}),((2,C),{(2,C,1)}),((3,A),{(3,A,1)})})");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/geo/GeoTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/geo/GeoTests.java b/datafu-pig/src/test/java/datafu/test/pig/geo/GeoTests.java
new file mode 100644
index 0000000..2e5f75e
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/geo/GeoTests.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.geo;
+
+import static org.testng.Assert.*;
+
+import java.util.List;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.test.pig.PigTests;
+
+public class GeoTests extends PigTests
+{
+ /**
+
+
+ define HaversineDistInMiles datafu.pig.geo.HaversineDistInMiles();
+
+ data = LOAD 'input' AS (lat1:double,lng1:double,lat2:double,lng2:double);
+
+ data2 = FOREACH data GENERATE HaversineDistInMiles(lat1,lng1,lat2,lng2);
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline
+ private String haversineTest;
+
+ @Test
+ public void haversineTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(haversineTest);
+
+ // Approximate latitude and longitude for major cities from maps.google.com
+ double[] la = {34.040143,-118.243103};
+ double[] tokyo = {35.637209,139.65271};
+ double[] ny = {40.716038,-73.99498};
+ double[] paris = {48.857713,2.342491};
+ double[] sydney = {-33.872696,151.195221};
+
+ this.writeLinesToFile("input",
+ coords(la,tokyo),
+ coords(ny,tokyo),
+ coords(ny,sydney),
+ coords(ny,paris));
+
+ test.runScript();
+
+ List<Tuple> distances = this.getLinesForAlias(test, "data2");
+
+ // ensure distance is within 20 miles of expected (distances found online)
+ assertWithin(5478.0, distances.get(0), 20.0); // la <-> tokyo
+ assertWithin(6760.0, distances.get(1), 20.0); // ny <-> tokyo
+ assertWithin(9935.0, distances.get(2), 20.0); // ny <-> sydney
+ assertWithin(3635.0, distances.get(3), 20.0); // ny <-> paris
+
+ }
+
+ private void assertWithin(double expected, Tuple actual, double maxDiff) throws Exception
+ {
+ Double actualVal = (Double)actual.get(0);
+ assertTrue(Math.abs(expected-actualVal) < maxDiff);
+ }
+
+ private String coords(double[] coords1, double[] coords2)
+ {
+ assertTrue(coords1.length == 2);
+ assertTrue(coords2.length == 2);
+ return String.format("%f\t%f\t%f\t%f", coords1[0], coords1[1], coords2[0], coords2[1]);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/hash/HashTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/hash/HashTests.java b/datafu-pig/src/test/java/datafu/test/pig/hash/HashTests.java
new file mode 100644
index 0000000..f422e39
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/hash/HashTests.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.hash;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.test.pig.PigTests;
+
+public class HashTests extends PigTests
+{
+ /**
+
+
+ define MD5 datafu.pig.hash.MD5();
+
+ data_in = LOAD 'input' as (val:chararray);
+
+ data_out = FOREACH data_in GENERATE MD5(val) as val;
+
+ STORE data_out INTO 'output';
+ */
+ @Multiline private String md5Test;
+
+ @Test
+ public void md5Test() throws Exception
+ {
+ PigTest test = createPigTestFromString(md5Test);
+
+ writeLinesToFile("input",
+ "ladsljkasdglk",
+ "lkadsljasgjskdjks",
+ "aladlasdgjks");
+
+ test.runScript();
+
+ assertOutput(test, "data_out",
+ "(d9a82575758bb4978949dc0659205cc6)",
+ "(9ec37f02fae0d8d6a7f4453a62272f1f)",
+ "(cb94139a8b9f3243e68a898ec6bd9b3d)");
+ }
+
+ /**
+
+
+ define SHA256 datafu.pig.hash.SHA('256');
+
+ data_in = LOAD 'input' as (val:chararray);
+
+ data_out = FOREACH data_in GENERATE SHA256(val) as val;
+
+ STORE data_out INTO 'output';
+ */
+ @Multiline private String sha256Test;
+
+ @Test
+ public void sha256Test() throws Exception
+ {
+ PigTest test = createPigTestFromString(sha256Test);
+
+ writeLinesToFile("input",
+ "ladsljkasdglk",
+ "lkadsljasgjskdjks",
+ "aladlasdgjks");
+
+ test.runScript();
+
+ assertOutput(test, "data_out",
+ "(70ebaf99c4d8ff8860869e50be2d46afbf150b883f66b50a76ee81cdc802242b)",
+ "(f22e3c744a9ade0fa591d28c55392035248b391c9ee4c77ebfeaf6558c8c0dac)",
+ "(49420b42e764178830783d4520aea56b759f325c1d1167f5640ded91f33f3e69)");
+ }
+
+ /**
+
+
+ define SHA512 datafu.pig.hash.SHA('512');
+
+ data_in = LOAD 'input' as (val:chararray);
+
+ data_out = FOREACH data_in GENERATE SHA512(val) as val;
+
+ STORE data_out INTO 'output';
+ */
+ @Multiline private String sha512Test;
+
+ @Test
+ public void sha512Test() throws Exception
+ {
+ PigTest test = createPigTestFromString(sha512Test);
+
+ writeLinesToFile("input",
+ "ladsljkasdglk",
+ "lkadsljasgjskdjks",
+ "aladlasdgjks");
+
+ test.runScript();
+
+ assertOutput(test, "data_out",
+ "(f681dbd89dfc9edf00f68107ed81b4b7c89abdf84337921785d13d9189937a43decbc264b5013d396a102b18564c39595732c43d6d4cc99473f6d6d7101ecf87)",
+ "(85c130c8636c052e52a2ca091a92d0bb98ee361adcbeeebbd6af978a593b2486a22ac1e7352c683035cfa28de8eee3402adc6760ad54c5c7eda122c5124766bd)",
+ "(3b82af5c08c9ab70523abf56db244eaa6740fa8c356e3a41bb5225560c0949b14b417c8d56e72cc26d5682400e0420a556e692c41ea82855013e8b7bae5fb0fb)");
+ }
+
+ /**
+
+
+ define MD5 datafu.pig.hash.MD5('base64');
+
+ data_in = LOAD 'input' as (val:chararray);
+
+ data_out = FOREACH data_in GENERATE MD5(val) as val;
+
+ STORE data_out INTO 'output';
+ */
+ @Multiline private String md5Base64Test;
+
+ @Test
+ public void md5Base64Test() throws Exception
+ {
+ PigTest test = createPigTestFromString(md5Base64Test);
+
+ writeLinesToFile("input",
+ "ladsljkasdglk",
+ "lkadsljasgjskdjks",
+ "aladlasdgjks");
+
+ test.runScript();
+
+ assertOutput(test, "data_out",
+ "(2agldXWLtJeJSdwGWSBcxg==)",
+ "(nsN/Avrg2Nan9EU6YicvHw==)",
+ "(y5QTmoufMkPmiomOxr2bPQ==)");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/linkanalysis/PageRankImplTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/linkanalysis/PageRankImplTests.java b/datafu-pig/src/test/java/datafu/test/pig/linkanalysis/PageRankImplTests.java
new file mode 100644
index 0000000..01d540f
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/linkanalysis/PageRankImplTests.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.linkanalysis;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.testng.annotations.Test;
+
+public class PageRankImplTests
+{
+ @Test
+ public void wikipediaGraphInMemoryTest() throws Exception {
+ System.out.println();
+ System.out.println("Starting wikipediaGraphInMemoryTest");
+
+ datafu.pig.linkanalysis.PageRankImpl graph = new datafu.pig.linkanalysis.PageRankImpl();
+
+ String[] edges = getWikiExampleEdges();
+
+ Map<String,Integer> nodeIdsMap = loadGraphFromEdgeList(graph, edges);
+
+ // Without dangling node handling we will not get the true page rank since the total rank will
+ // not add to 1.0. Without dangling node handling some of the page rank drains out of the graph.
+ graph.enableDanglingNodeHandling();
+
+ performIterations(graph, 150, 1e-18f);
+
+ String[] expectedRanks = getWikiExampleExpectedRanks();
+
+ Map<String,Float> expectedRanksMap = parseExpectedRanks(expectedRanks);
+
+ validateExpectedRanks(graph, nodeIdsMap, expectedRanksMap);
+ }
+
+ @Test
+ public void wikipediaGraphDiskCacheTest() throws Exception {
+ System.out.println();
+ System.out.println("Starting wikipediaGraphDiskCacheTest");
+
+ datafu.pig.linkanalysis.PageRankImpl graph = new datafu.pig.linkanalysis.PageRankImpl();
+
+ String[] edges = getWikiExampleEdges();
+
+ graph.enableEdgeDiskCaching();
+ graph.setEdgeCachingThreshold(5);
+
+ Map<String,Integer> nodeIdsMap = loadGraphFromEdgeList(graph, edges);
+
+ assert graph.isUsingEdgeDiskCache() : "Expected disk cache to be used";
+
+ // Without dangling node handling we will not get the true page rank since the total rank will
+ // not add to 1.0. Without dangling node handling some of the page rank drains out of the graph.
+ graph.enableDanglingNodeHandling();
+
+ performIterations(graph, 150, 1e-18f);
+
+ String[] expectedRanks = getWikiExampleExpectedRanks();
+
+ Map<String,Float> expectedRanksMap = parseExpectedRanks(expectedRanks);
+
+ validateExpectedRanks(graph, nodeIdsMap, expectedRanksMap);
+ }
+
+ @Test(groups="perf")
+ public void hubAndSpokeInMemoryTest() throws Exception {
+ System.out.println();
+ System.out.println("Starting hubAndSpokeInMemoryTest");
+
+ datafu.pig.linkanalysis.PageRankImpl graph = new datafu.pig.linkanalysis.PageRankImpl();
+
+ String[] edges = getHubAndSpokeEdges();
+
+ Map<String,Integer> nodeIdsMap = loadGraphFromEdgeList(graph, edges);
+
+ graph.enableDanglingNodeHandling();
+
+ performIterations(graph, 150, 1e-18f);
+
+ // no need to validate, this is just a perf test for runtime comparison
+ }
+
+ @Test(groups="perf")
+ public void hubAndSpokeDiskCacheTest() throws Exception {
+ System.out.println();
+ System.out.println("Starting hubAndSpokeDiskCacheTest");
+
+ datafu.pig.linkanalysis.PageRankImpl graph = new datafu.pig.linkanalysis.PageRankImpl();
+
+ String[] edges = getHubAndSpokeEdges();
+
+ graph.enableEdgeDiskCaching();
+ graph.setEdgeCachingThreshold(5);
+
+ Map<String,Integer> nodeIdsMap = loadGraphFromEdgeList(graph, edges);
+
+ graph.enableDanglingNodeHandling();
+
+ performIterations(graph, 150, 1e-18f);
+
+ // no need to validate, this is just a perf test for runtime comparison
+ }
+
+ private String[] getHubAndSpokeEdges()
+ {
+ int count = 50000;
+ String[] edges = new String[count];
+
+ for (int i=0; i<count; i++)
+ {
+ edges[i] = String.format("S%d H", i);
+ }
+ return edges;
+ }
+
+ public static String[] getWikiExampleEdges()
+ {
+ // graph taken from:
+ // http://en.wikipedia.org/wiki/PageRank
+ String[] edges = {
+ "B C",
+ "C B",
+ "D A",
+ "D B",
+ "E D",
+ "E B",
+ "E F",
+ "F E",
+ "F B",
+ "P1 B",
+ "P1 E",
+ "P2 B",
+ "P2 E",
+ "P3 B",
+ "P3 E",
+ "P4 E",
+ "P5 E"
+ };
+ return edges;
+ }
+
+ public static String[] getWikiExampleExpectedRanks()
+ {
+ // these ranks come from the Wikipedia page:
+ // http://en.wikipedia.org/wiki/PageRank
+ String[] expectedRanks = {
+ "A 3.3",
+ "B 38.4",
+ "C 34.3",
+ "D 3.9",
+ "E 8.1",
+ "F 3.9",
+ "P1 1.6",
+ "P2 1.6",
+ "P3 1.6",
+ "P4 1.6",
+ "P5 1.6"
+ };
+ return expectedRanks;
+ }
+
+ private Map<String,Integer> loadGraphFromEdgeList(datafu.pig.linkanalysis.PageRankImpl graph, String[] edges) throws IOException
+ {
+ Map<Integer,ArrayList<Map<String,Object>>> nodeEdgesMap = new HashMap<Integer,ArrayList<Map<String,Object>>>();
+ Map<String,Integer> nodeIdsMap = new HashMap<String,Integer>();
+
+ for (String edge : edges)
+ {
+ String[] parts = edge.split(" ");
+ assert parts.length == 2 : "Expected two parts";
+
+ int sourceId = getOrCreateId(parts[0], nodeIdsMap);
+ int destId = getOrCreateId(parts[1], nodeIdsMap);
+
+ Map<String,Object> edgeMap = new HashMap<String,Object>();
+ edgeMap.put("weight", 1.0);
+ edgeMap.put("dest", destId);
+
+ ArrayList<Map<String,Object>> nodeEdges = null;
+
+ if (nodeEdgesMap.containsKey(sourceId))
+ {
+ nodeEdges = nodeEdgesMap.get(sourceId);
+ }
+ else
+ {
+ nodeEdges = new ArrayList<Map<String,Object>>();
+ nodeEdgesMap.put(sourceId, nodeEdges);
+ }
+
+ nodeEdges.add(edgeMap);
+ }
+
+ for (Map.Entry<Integer, ArrayList<Map<String,Object>>> e : nodeEdgesMap.entrySet())
+ {
+ graph.addNode(e.getKey(), e.getValue());
+ }
+
+ return nodeIdsMap;
+ }
+
+ private void performIterations(datafu.pig.linkanalysis.PageRankImpl graph, int maxIters, float tolerance) throws IOException
+ {
+ System.out.println(String.format("Beginning iteration (maxIters = %d, tolerance=%e)", maxIters, tolerance));
+
+ System.out.println("Initializing graph");
+ long startTime = System.nanoTime();
+ graph.init();
+ System.out.println(String.format("Done, took %f ms", (System.nanoTime() - startTime)/10.0e6));
+
+ float totalDiff;
+ int iter = 0;
+
+ System.out.println("Beginning iterations");
+ startTime = System.nanoTime();
+ do
+ {
+ totalDiff = graph.nextIteration();
+ iter++;
+ } while(iter < maxIters && totalDiff > tolerance);
+ System.out.println(String.format("Done, took %f ms", (System.nanoTime() - startTime)/10.0e6));
+ }
+
+ private void validateExpectedRanks(datafu.pig.linkanalysis.PageRankImpl graph, Map<String,Integer> nodeIds, Map<String,Float> expectedRanks)
+ {
+ System.out.println("Validating page rank results");
+
+ for (Map.Entry<String,Integer> e : nodeIds.entrySet())
+ {
+ float rank = graph.getNodeRank(e.getValue());
+
+ float expectedRank = expectedRanks.get(e.getKey());
+ // require 0.1% accuracy
+ assert (Math.abs(expectedRank - rank*100.0f) < 0.1) : String.format("Did not get expected rank for %s", e.getKey());
+ }
+
+ System.out.println("All ranks match expected");
+ }
+
+ public static Map<String,Float> parseExpectedRanks(String[] expectedRanks)
+ {
+ Map<String,Float> expectedRanksMap = new HashMap<String,Float>();
+ for (String expectedRankString : expectedRanks)
+ {
+ String[] parts = expectedRankString.split(" ");
+ assert parts.length == 2 : "Expected two parts";
+ String name = parts[0];
+ Float expectedRank = Float.parseFloat(parts[1]);
+ expectedRanksMap.put(name, expectedRank);
+ }
+ return expectedRanksMap;
+ }
+
+ private Integer getOrCreateId(String name, Map<String,Integer> nodeIds)
+ {
+ if (nodeIds.containsKey(name))
+ {
+ return nodeIds.get(name);
+ }
+ else
+ {
+ Integer id = nodeIds.size();
+ nodeIds.put(name, id);
+ return id;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/linkanalysis/PageRankTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/linkanalysis/PageRankTests.java b/datafu-pig/src/test/java/datafu/test/pig/linkanalysis/PageRankTests.java
new file mode 100644
index 0000000..f48038c
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/linkanalysis/PageRankTests.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.linkanalysis;
+
+
+import static org.testng.Assert.*;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+
+import datafu.test.pig.linkanalysis.PageRankImplTests;
+import datafu.test.pig.PigTests;
+
+public class PageRankTests extends PigTests
+{
+ /**
+
+
+ -- Need to enable dangling node handling since the Wikipedia example has them,
+ -- otherwise the ranks won't be right.
+ define PageRank datafu.pig.linkanalysis.PageRank('dangling_nodes','true');
+
+ data = LOAD 'input' AS (topic:INT,source:INT,dest:INT,weight:DOUBLE);
+
+ data_grouped = GROUP data by (topic,source);
+
+ data_grouped = foreach data_grouped {
+ generate group.topic as topic, group.source as source, data.(dest,weight) as edges;
+ };
+
+ data_grouped2 = GROUP data_grouped by topic;
+ data_grouped2 = foreach data_grouped2 {
+ generate group as topic, FLATTEN(PageRank(data_grouped.(source,edges))) as (source,rnk);
+ };
+
+ data_grouped3 = FOREACH data_grouped2 GENERATE
+ topic,
+ source,
+ rnk;
+
+ STORE data_grouped3 INTO 'output';
+
+
+ */
+ @Multiline private String pageRankTest;
+
+ @Test
+ public void pigPageRankTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(pageRankTest);
+
+ String[] edges = PageRankImplTests.getWikiExampleEdges();
+
+ Map<String,Integer> nodeIds = new HashMap<String,Integer>();
+ Map<Integer,String> nodeIdsReversed = new HashMap<Integer,String>();
+ Map<String,Float> expectedRanks = PageRankImplTests.parseExpectedRanks(PageRankImplTests.getWikiExampleExpectedRanks());
+
+ File f = new File(System.getProperty("user.dir"), "input").getAbsoluteFile();
+ if (f.exists())
+ {
+ f.delete();
+ }
+
+ FileWriter writer = new FileWriter(f);
+ BufferedWriter bufferedWriter = new BufferedWriter(writer);
+
+ for (String edge : edges)
+ {
+ String[] edgeParts = edge.split(" ");
+ String source = edgeParts[0];
+ String dest = edgeParts[1];
+ if (!nodeIds.containsKey(source))
+ {
+ int id = nodeIds.size();
+ nodeIds.put(source,id);
+ nodeIdsReversed.put(id, source);
+ }
+ if (!nodeIds.containsKey(dest))
+ {
+ int id = nodeIds.size();
+ nodeIds.put(dest,id);
+ nodeIdsReversed.put(id, dest);
+ }
+ Integer sourceId = nodeIds.get(source);
+ Integer destId = nodeIds.get(dest);
+
+ StringBuffer sb = new StringBuffer();
+
+ sb.append("1\t"); // topic
+ sb.append(sourceId.toString() + "\t");
+ sb.append(destId.toString() + "\t");
+ sb.append("1.0\n"); // weight
+
+ bufferedWriter.write(sb.toString());
+ }
+
+ bufferedWriter.close();
+
+ test.runScript();
+ Iterator<Tuple> tuples = test.getAlias("data_grouped3");
+
+ System.out.println("Final node ranks:");
+ int nodeCount = 0;
+ while (tuples.hasNext())
+ {
+ Tuple nodeTuple = tuples.next();
+
+ Integer topic = (Integer)nodeTuple.get(0);
+ Integer nodeId = (Integer)nodeTuple.get(1);
+ Float nodeRank = (Float)nodeTuple.get(2);
+
+ assertEquals(1, topic.intValue());
+
+ System.out.println(String.format("%d => %f", nodeId, nodeRank));
+
+ Float expectedNodeRank = expectedRanks.get(nodeIdsReversed.get(nodeId));
+
+ assertTrue(Math.abs(expectedNodeRank - nodeRank * 100.0f) < 0.1,
+ String.format("expected: %f, actual: %f", expectedNodeRank, nodeRank));
+
+ nodeCount++;
+ }
+
+ assertEquals(nodeIds.size(),nodeCount);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/random/NumberTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/random/NumberTests.java b/datafu-pig/src/test/java/datafu/test/pig/random/NumberTests.java
new file mode 100644
index 0000000..e2782c4
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/random/NumberTests.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.random;
+
+import static org.testng.Assert.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.test.pig.PigTests;
+
+public class NumberTests extends PigTests
+{
+ /**
+
+
+ define RandInt datafu.pig.random.RandInt();
+
+ data = LOAD 'input' AS (key:INT);
+ data2 = FOREACH data GENERATE key, RandInt($MIN,$MAX) as val;
+
+ STORE data2 INTO 'output';
+
+ */
+ @Multiline private String randomIntRangeTest;
+
+ /**
+ * Test the RandomIntRange UDF. The main purpose is to make sure it can be used in a Pig script.
+ * Also the range of output values is tested.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void randomIntRangeTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(randomIntRangeTest,
+ "MIN=1", "MAX=10");
+
+ List<String> input = new ArrayList<String>();
+ for (int i=0; i<100; i++)
+ {
+ input.add(String.format("(%d)", i));
+ }
+
+ writeLinesToFile("input",
+ input.toArray(new String[0]));
+
+ test.runScript();
+
+ List<Tuple> tuples = getLinesForAlias(test, "data2", false);
+ for (Tuple tuple : tuples)
+ {
+ Integer randValue = (Integer)tuple.get(1);
+ assertTrue(randValue >= 1);
+ assertTrue(randValue <= 10);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/random/UUIDTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/random/UUIDTests.java b/datafu-pig/src/test/java/datafu/test/pig/random/UUIDTests.java
new file mode 100644
index 0000000..ab2002d
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/random/UUIDTests.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package datafu.test.pig.random;
+
+import datafu.test.pig.PigTests;
+import junit.framework.Assert;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import java.util.*;
+
+import static org.testng.Assert.assertTrue;
+
+public class UUIDTests extends PigTests
+{
+ /**
+
+ define RandomUUID datafu.pig.random.RandomUUID();
+
+ data = LOAD 'input' AS (key: chararray);
+ DUMP data
+
+ data2 = FOREACH data GENERATE key, RandomUUID() as val;
+ DUMP data2
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline private String randomUUIDTest;
+
+ /**
+ * Test the RandomUUID UDF. The main purpose is to make sure it can be used in a Pig script.
+ * Also the range of length of values is tested.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void randomUUIDTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(randomUUIDTest);
+
+ writeLinesToFile("input",
+ "input1",
+ "input2",
+ "input3");
+
+ List<Tuple> tuples = getLinesForAlias(test, "data2", true);
+ Set<UUID> set = new HashSet<UUID>();
+ for (Tuple tuple : tuples)
+ {
+ set.add(UUID.fromString((String)tuple.get(1)));
+ }
+ Assert.assertEquals(set.size(), 3);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/sampling/SamplingTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/sampling/SamplingTests.java b/datafu-pig/src/test/java/datafu/test/pig/sampling/SamplingTests.java
new file mode 100644
index 0000000..2f685d4
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/sampling/SamplingTests.java
@@ -0,0 +1,523 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.sampling;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import junit.framework.Assert;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.pig.sampling.ReservoirSample;
+import datafu.pig.sampling.SampleByKey;
+import datafu.pig.sampling.WeightedSample;
+import datafu.test.pig.PigTests;
+
+
+public class SamplingTests extends PigTests
+{
+ /**
+
+
+ define WeightedSample datafu.pig.sampling.WeightedSample('1');
+
+ data = LOAD 'input' AS (A: bag {T: tuple(v1:chararray,v2:INT)});
+
+ data2 = FOREACH data GENERATE WeightedSample(A,1);
+ --describe data2;
+
+ STORE data2 INTO 'output';
+
+ */
+ @Multiline
+ private String weightedSampleTest;
+
+ @Test
+ public void weightedSampleTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(weightedSampleTest);
+
+ writeLinesToFile("input",
+ "({(a, 100),(b, 1),(c, 5),(d, 2)})");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "({(a,100),(c,5),(b,1),(d,2)})");
+ }
+
+ /**
+
+
+ define WeightedSample datafu.pig.sampling.WeightedSample('1');
+
+ data = LOAD 'input' AS (A: bag {T: tuple(v1:chararray,v2:INT)});
+
+ data2 = FOREACH data GENERATE WeightedSample(A,1,3);
+ --describe data2;
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline
+ private String weightedSampleLimitTest;
+
+ @Test
+ public void weightedSampleLimitTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(weightedSampleLimitTest);
+
+ writeLinesToFile("input",
+ "({(a, 100),(b, 1),(c, 5),(d, 2)})");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "({(a,100),(c,5),(b,1)})");
+ }
+
+ @Test
+ public void weightedSampleLimitExecTest() throws IOException
+ {
+ WeightedSample sampler = new WeightedSample();
+
+ DataBag bag = BagFactory.getInstance().newDefaultBag();
+ for (int i=0; i<100; i++)
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(2);
+ t.set(0, i);
+ t.set(1, 1); // score is equal for all
+ bag.add(t);
+ }
+
+ Tuple input = TupleFactory.getInstance().newTuple(3);
+ input.set(0, bag);
+ input.set(1, 1); // use index 1 for score
+ input.set(2, 10); // get 10 items
+
+ DataBag result = sampler.exec(input);
+
+ Assert.assertEquals(10, result.size());
+
+ // all must be found, no repeats
+ Set<Integer> found = new HashSet<Integer>();
+ for (Tuple t : result)
+ {
+ Integer i = (Integer)t.get(0);
+ System.out.println(i);
+ Assert.assertTrue(i>=0 && i<100);
+ Assert.assertFalse(String.format("Found duplicate of %d",i), found.contains(i));
+ found.add(i);
+ }
+ }
+
+ /**
+
+
+ DEFINE SampleByKey datafu.pig.sampling.SampleByKey('0.5', 'salt2.5');
+
+ data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
+ sampled = FILTER data BY SampleByKey(A_id);
+
+ STORE sampled INTO 'output';
+
+ */
+ @Multiline
+ private String sampleByKeyTest;
+
+ @Test
+ public void sampleByKeyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(sampleByKeyTest);
+
+ writeLinesToFile("input",
+ "A1\tB1\t1","A1\tB1\t4","A1\tB3\t4","A1\tB4\t4",
+ "A2\tB1\t4","A2\tB2\t4",
+ "A3\tB1\t3","A3\tB1\t1","A3\tB3\t77",
+ "A4\tB1\t3","A4\tB2\t3","A4\tB3\t59","A4\tB4\t29",
+ "A5\tB1\t4",
+ "A6\tB2\t3","A6\tB2\t55","A6\tB3\t1",
+ "A7\tB1\t39","A7\tB2\t27","A7\tB3\t85",
+ "A8\tB1\t4","A8\tB2\t45",
+ "A9\tB3\t92", "A9\tB1\t42","A9\tB2\t1","A9\tB3\t0",
+ "A10\tB1\t7","A10\tB2\t23","A10\tB3\t1","A10\tB4\t41","A10\tB5\t52");
+
+ test.runScript();
+ assertOutput(test, "sampled",
+ "(A4,B1,3)","(A4,B2,3)","(A4,B3,59)","(A4,B4,29)",
+ "(A5,B1,4)",
+ "(A6,B2,3)","(A6,B2,55)","(A6,B3,1)",
+ "(A7,B1,39)","(A7,B2,27)","(A7,B3,85)",
+ "(A10,B1,7)","(A10,B2,23)","(A10,B3,1)","(A10,B4,41)","(A10,B5,52)");
+ }
+
+ /**
+
+
+ DEFINE SampleByKey datafu.pig.sampling.SampleByKey('0.5', 'salt2.5');
+
+ data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
+ sampled = FILTER data BY SampleByKey(A_id, B_id);
+
+ STORE sampled INTO 'output';
+
+ */
+ @Multiline
+ private String sampleByKeyMultipleKeyTest;
+
+ @Test
+ public void sampleByKeyMultipleKeyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(sampleByKeyMultipleKeyTest);
+
+ writeLinesToFile("input",
+ "A1\tB1\t1","A1\tB1\t4",
+ "A1\tB3\t4",
+ "A1\tB4\t4",
+ "A2\tB1\t4",
+ "A2\tB2\t4",
+ "A3\tB1\t3","A3\tB1\t1",
+ "A3\tB3\t77",
+ "A4\tB1\t3",
+ "A4\tB2\t3",
+ "A4\tB3\t59",
+ "A4\tB4\t29",
+ "A5\tB1\t4",
+ "A6\tB2\t3","A6\tB2\t55",
+ "A6\tB3\t1",
+ "A7\tB1\t39",
+ "A7\tB2\t27",
+ "A7\tB3\t85",
+ "A8\tB1\t4",
+ "A8\tB2\t45",
+ "A9\tB3\t92","A9\tB3\t0",
+ "A9\tB6\t42","A9\tB5\t1",
+ "A10\tB1\t7",
+ "A10\tB2\t23","A10\tB2\t1","A10\tB2\t31",
+ "A10\tB6\t41",
+ "A10\tB7\t52");
+ test.runScript();
+ assertOutput(test, "sampled",
+ "(A1,B1,1)","(A1,B1,4)",
+ "(A1,B4,4)",
+ "(A2,B1,4)",
+ "(A2,B2,4)",
+ "(A3,B1,3)","(A3,B1,1)",
+ "(A4,B4,29)",
+ "(A5,B1,4)",
+ "(A6,B3,1)",
+ "(A7,B1,39)",
+ "(A8,B1,4)",
+ "(A9,B3,92)","(A9,B3,0)",
+ "(A10,B2,23)","(A10,B2,1)","(A10,B2,31)"
+ );
+
+ }
+
+ @Test
+ public void sampleByKeyExecTest() throws Exception
+ {
+ SampleByKey sampler = new SampleByKey("0.10", "thesalt");
+
+ Map<Integer,Integer> valuesPerKey = new HashMap<Integer,Integer>();
+
+ // 10,000 keys total
+ for (int i=0; i<10000; i++)
+ {
+ // 5 values per key
+ for (int j=0; j<5; j++)
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, i);
+ if (sampler.exec(t))
+ {
+ if (valuesPerKey.containsKey(i))
+ {
+ valuesPerKey.put(i, valuesPerKey.get(i)+1);
+ }
+ else
+ {
+ valuesPerKey.put(i, 1);
+ }
+ }
+ }
+ }
+
+ // 10% sample, so should have roughly 1000 keys
+ Assert.assertTrue(Math.abs(1000-valuesPerKey.size()) < 50);
+
+ // every value should be present for the same key
+ for (Map.Entry<Integer, Integer> pair : valuesPerKey.entrySet())
+ {
+ Assert.assertEquals(5, pair.getValue().intValue());
+ }
+ }
+
+ /**
+
+
+ DEFINE ReservoirSample datafu.pig.sampling.ReservoirSample('$RESERVOIR_SIZE');
+
+ data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
+ sampled = FOREACH (GROUP data ALL) GENERATE ReservoirSample(data) as sample_data;
+ sampled = FOREACH sampled GENERATE COUNT(sample_data) AS sample_count;
+ STORE sampled INTO 'output';
+
+ */
+ @Multiline
+ private String reservoirSampleTest;
+
+ @Test
+ public void reservoirSampleTest() throws Exception
+ {
+
+ writeLinesToFile("input",
+ "A1\tB1\t1",
+ "A1\tB1\t4",
+ "A1\tB3\t4",
+ "A1\tB4\t4",
+ "A2\tB1\t4",
+ "A2\tB2\t4",
+ "A3\tB1\t3",
+ "A3\tB1\t1",
+ "A3\tB3\t77",
+ "A4\tB1\t3",
+ "A4\tB2\t3",
+ "A4\tB3\t59",
+ "A4\tB4\t29",
+ "A5\tB1\t4",
+ "A6\tB2\t3",
+ "A6\tB2\t55",
+ "A6\tB3\t1",
+ "A7\tB1\t39",
+ "A7\tB2\t27",
+ "A7\tB3\t85",
+ "A8\tB1\t4",
+ "A8\tB2\t45",
+ "A9\tB3\t92",
+ "A9\tB3\t0",
+ "A9\tB6\t42",
+ "A9\tB5\t1",
+ "A10\tB1\t7",
+ "A10\tB2\t23",
+ "A10\tB2\t1",
+ "A10\tB2\t31",
+ "A10\tB6\t41",
+ "A10\tB7\t52");
+
+ for(int i=10; i<=30; i=i+10){
+ int reservoirSize = i ;
+ PigTest test = createPigTestFromString(reservoirSampleTest, "RESERVOIR_SIZE="+reservoirSize);
+ test.runScript();
+ assertOutput(test, "sampled", "("+reservoirSize+")");
+ }
+ }
+
+ /**
+
+
+ DEFINE ReservoirSample datafu.pig.sampling.ReservoirSample('$RESERVOIR_SIZE');
+ DEFINE Assert datafu.pig.util.Assert();
+
+ data = LOAD 'input' AS (A_id:int, B_id:chararray, C:int);
+ sampled = FOREACH (GROUP data BY A_id) GENERATE group as A_id, ReservoirSample(data.(B_id,C)) as sample_data;
+ sampled = FILTER sampled BY Assert((SIZE(sample_data) <= $RESERVOIR_SIZE ? 1 : 0), 'must be <= $RESERVOIR_SIZE');
+ sampled = FOREACH sampled GENERATE A_id, FLATTEN(sample_data);
+ STORE sampled INTO 'output';
+
+ */
+ @Multiline
+ private String reservoirSampleGroupTest;
+
+ /**
+ * Verifies that ReservoirSample works when data grouped by a key.
+ * In particular it ensures that the reservoir is not reused across keys.
+ *
+ * <p>
+ * This confirms the fix for DATAFU-11.
+ * </p>
+ *
+ * @throws Exception
+ */
+ @Test
+ public void reservoirSampleGroupTest() throws Exception
+ {
+ // first value is the key. last value matches the key so we can
+ // verify the register is reset for each key. values should not
+ // bleed across to other keys.
+ writeLinesToFile("input",
+ "1\tB1\t1",
+ "1\tB1\t1",
+ "1\tB3\t1",
+ "1\tB4\t1",
+ "2\tB1\t2",
+ "2\tB2\t2",
+ "3\tB1\t3",
+ "3\tB1\t3",
+ "3\tB3\t3",
+ "4\tB1\t4",
+ "4\tB2\t4",
+ "4\tB3\t4",
+ "4\tB4\t4",
+ "5\tB1\t5",
+ "6\tB2\t6",
+ "6\tB2\t6",
+ "6\tB3\t6",
+ "7\tB1\t7",
+ "7\tB2\t7",
+ "7\tB3\t7",
+ "8\tB1\t8",
+ "8\tB2\t8",
+ "9\tB3\t9",
+ "9\tB3\t9",
+ "9\tB6\t9",
+ "9\tB5\t9",
+ "10\tB1\t10",
+ "10\tB2\t10",
+ "10\tB2\t10",
+ "10\tB2\t10",
+ "10\tB6\t10",
+ "10\tB7\t10");
+
+ for(int i=1; i<=3; i++) {
+ int reservoirSize = i ;
+ PigTest test = createPigTestFromString(reservoirSampleGroupTest, "RESERVOIR_SIZE="+reservoirSize);
+ test.runScript();
+
+ List<Tuple> tuples = getLinesForAlias(test, "sampled");
+
+ for (Tuple tuple : tuples)
+ {
+ Assert.assertEquals(((Number)tuple.get(0)).intValue(), ((Number)tuple.get(2)).intValue());
+ }
+ }
+ }
+
+ @Test
+ public void reservoirSampleExecTest() throws IOException
+ {
+ ReservoirSample sampler = new ReservoirSample("10");
+
+ DataBag bag = BagFactory.getInstance().newDefaultBag();
+ for (int i=0; i<100; i++)
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, i);
+ bag.add(t);
+ }
+
+ Tuple input = TupleFactory.getInstance().newTuple(bag);
+
+ DataBag result = sampler.exec(input);
+
+ Assert.assertEquals(10, result.size());
+
+ // all must be found, no repeats
+ Set<Integer> found = new HashSet<Integer>();
+ for (Tuple t : result)
+ {
+ Integer i = (Integer)t.get(0);
+ System.out.println(i);
+ Assert.assertTrue(i>=0 && i<100);
+ Assert.assertFalse(String.format("Found duplicate of %d",i), found.contains(i));
+ found.add(i);
+ }
+ }
+
+ @Test
+ public void reservoirSampleAccumulateTest() throws IOException
+ {
+ ReservoirSample sampler = new ReservoirSample("10");
+
+ for (int i=0; i<100; i++)
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, i);
+ DataBag bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(t);
+ Tuple input = TupleFactory.getInstance().newTuple(bag);
+ sampler.accumulate(input);
+ }
+
+ DataBag result = sampler.getValue();
+
+ Assert.assertEquals(10, result.size());
+
+ // all must be found, no repeats
+ Set<Integer> found = new HashSet<Integer>();
+ for (Tuple t : result)
+ {
+ Integer i = (Integer)t.get(0);
+ System.out.println(i);
+ Assert.assertTrue(i>=0 && i<100);
+ Assert.assertFalse(String.format("Found duplicate of %d",i), found.contains(i));
+ found.add(i);
+ }
+ }
+
+ @Test
+ public void reservoirSampleAlgebraicTest() throws IOException
+ {
+ ReservoirSample.Initial initialSampler = new ReservoirSample.Initial("10");
+ ReservoirSample.Intermediate intermediateSampler = new ReservoirSample.Intermediate("10");
+ ReservoirSample.Final finalSampler = new ReservoirSample.Final("10");
+
+ DataBag bag = BagFactory.getInstance().newDefaultBag();
+ for (int i=0; i<100; i++)
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, i);
+ bag.add(t);
+ }
+
+ Tuple input = TupleFactory.getInstance().newTuple(bag);
+
+ Tuple intermediateTuple = initialSampler.exec(input);
+ DataBag intermediateBag = BagFactory.getInstance().newDefaultBag(Arrays.asList(intermediateTuple));
+ intermediateTuple = intermediateSampler.exec(TupleFactory.getInstance().newTuple(intermediateBag));
+ intermediateBag = BagFactory.getInstance().newDefaultBag(Arrays.asList(intermediateTuple));
+ DataBag result = finalSampler.exec(TupleFactory.getInstance().newTuple(intermediateBag));
+
+ Assert.assertEquals(10, result.size());
+
+ // all must be found, no repeats
+ Set<Integer> found = new HashSet<Integer>();
+ for (Tuple t : result)
+ {
+ Integer i = (Integer)t.get(0);
+ System.out.println(i);
+ Assert.assertTrue(i>=0 && i<100);
+ Assert.assertFalse(String.format("Found duplicate of %d",i), found.contains(i));
+ found.add(i);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/sampling/SimpleRandomSampleTest.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/sampling/SimpleRandomSampleTest.java b/datafu-pig/src/test/java/datafu/test/pig/sampling/SimpleRandomSampleTest.java
new file mode 100644
index 0000000..41d9c02
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/sampling/SimpleRandomSampleTest.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.sampling;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.pig.sampling.SimpleRandomSample;
+import datafu.test.pig.PigTests;
+
+/**
+ * Tests for {@link SimpleRandomSample}.
+ *
+ * @author ximeng
+ *
+ */
+public class SimpleRandomSampleTest extends PigTests
+{
+ /**
+ *
+ *
+ * DEFINE SRS datafu.pig.sampling.SimpleRandomSample();
+ *
+ * data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
+ *
+ * sampled = FOREACH (GROUP data ALL) GENERATE SRS(data, $p) as sample_data;
+ *
+ * sampled = FOREACH sampled GENERATE COUNT(sample_data) AS sample_count;
+ *
+ * STORE sampled INTO 'output';
+ */
+ @Multiline
+ private String simpleRandomSampleTest;
+
+ /**
+ *
+ *
+ * DEFINE SRS datafu.pig.sampling.SimpleRandomSample();
+ *
+ * data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
+ *
+ * sampled = FOREACH (GROUP data ALL) GENERATE SRS(data, $p, $n1) as sample_data;
+ *
+ * sampled = FOREACH sampled GENERATE COUNT(sample_data) AS sample_count;
+ *
+ * STORE sampled INTO 'output';
+ */
+ @Multiline
+ private String simpleRandomSampleWithLowerBoundTest;
+
+ /**
+ *
+ *
+ * DEFINE SRS datafu.pig.sampling.SimpleRandomSample();
+ *
+ * data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
+ *
+ * sampled = FOREACH (GROUP data ALL) GENERATE SRS(data, $p1) as sample_1, SRS(data,
+ * $p2) AS sample_2;
+ *
+ * sampled = FOREACH sampled GENERATE COUNT(sample_1) AS sample_count_1, COUNT(sample_2)
+ * AS sample_count_2;
+ *
+ * STORE sampled INTO 'output';
+ */
+ @Multiline
+ private String simpleRandomSampleWithTwoCallsTest;
+
+ @Test
+ public void testSimpleRandomSample() throws Exception
+ {
+ writeLinesToFile("input",
+ "A1\tB1\t1",
+ "A1\tB1\t4",
+ "A1\tB3\t4",
+ "A1\tB4\t4",
+ "A2\tB1\t4",
+ "A2\tB2\t4",
+ "A3\tB1\t3",
+ "A3\tB1\t1",
+ "A3\tB3\t77",
+ "A4\tB1\t3",
+ "A4\tB2\t3",
+ "A4\tB3\t59",
+ "A4\tB4\t29",
+ "A5\tB1\t4",
+ "A6\tB2\t3",
+ "A6\tB2\t55",
+ "A6\tB3\t1",
+ "A7\tB1\t39",
+ "A7\tB2\t27",
+ "A7\tB3\t85",
+ "A8\tB1\t4",
+ "A8\tB2\t45",
+ "A9\tB3\t92",
+ "A9\tB3\t0",
+ "A9\tB6\t42",
+ "A9\tB5\t1",
+ "A10\tB1\t7",
+ "A10\tB2\t23",
+ "A10\tB2\t1",
+ "A10\tB2\t31",
+ "A10\tB6\t41",
+ "A10\tB7\t52");
+
+ int n = 32;
+ double p = 0.3;
+ int s = (int) Math.ceil(p * n);
+
+ PigTest test = createPigTestFromString(simpleRandomSampleTest, "p=" + p);
+ test.runScript();
+ assertOutput(test, "sampled", "(" + s + ")");
+
+ int n1 = 30;
+ PigTest testWithLB =
+ createPigTestFromString(simpleRandomSampleWithLowerBoundTest, "p=" + p, "n1="
+ + n1);
+ testWithLB.runScript();
+ assertOutput(testWithLB, "sampled", "(" + s + ")");
+
+ double p1 = 0.05;
+ double p2 = 0.95;
+ int s1 = (int) Math.ceil(p1 * n);
+ int s2 = (int) Math.ceil(p2 * n);
+
+ PigTest testWithTwoCalls =
+ createPigTestFromString(simpleRandomSampleWithTwoCallsTest, "p1=" + p1, "p2="
+ + p2);
+ testWithTwoCalls.runScript();
+ assertOutput(testWithTwoCalls, "sampled", "(" + s1 + "," + s2 + ")");
+
+ test.runScript();
+
+ }
+
+ /**
+ *
+ *
+ * DEFINE SRS datafu.pig.sampling.SimpleRandomSample();
+ *
+ * data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
+ *
+ * sampled = FOREACH (GROUP data BY A_id) GENERATE group, SRS(data,
+ * $SAMPLING_PROBABILITY) as sample_data;
+ *
+ * sampled = FOREACH sampled GENERATE group, COUNT(sample_data) AS sample_count;
+ *
+ * sampled = ORDER sampled BY group;
+ *
+ * STORE sampled INTO 'output';
+ */
+ @Multiline
+ private String stratifiedSampleTest;
+
+ @Test
+ public void testStratifiedSample() throws Exception
+ {
+ writeLinesToFile("input",
+ "A1\tB1\t1",
+ "A1\tB1\t4",
+ "A1\tB3\t4",
+ "A1\tB4\t4",
+ "A2\tB1\t4",
+ "A2\tB2\t4",
+ "A3\tB1\t3",
+ "A3\tB1\t1",
+ "A3\tB3\t77",
+ "A4\tB1\t3",
+ "A4\tB2\t3",
+ "A4\tB3\t59",
+ "A4\tB4\t29",
+ "A5\tB1\t4",
+ "A6\tB2\t3",
+ "A6\tB2\t55",
+ "A6\tB3\t1",
+ "A7\tB1\t39",
+ "A7\tB2\t27",
+ "A7\tB3\t85",
+ "A8\tB1\t4",
+ "A8\tB2\t45",
+ "A9\tB3\t92",
+ "A9\tB3\t0",
+ "A9\tB6\t42",
+ "A9\tB5\t1",
+ "A10\tB1\t7",
+ "A10\tB2\t23",
+ "A10\tB2\t1",
+ "A10\tB2\t31",
+ "A10\tB6\t41",
+ "A10\tB7\t52");
+
+ double p = 0.5;
+
+ PigTest test =
+ createPigTestFromString(stratifiedSampleTest, "SAMPLING_PROBABILITY=" + p);
+ test.runScript();
+ assertOutput(test,
+ "sampled",
+ "(A1,2)",
+ "(A10,3)",
+ "(A2,1)",
+ "(A3,2)",
+ "(A4,2)",
+ "(A5,1)",
+ "(A6,2)",
+ "(A7,2)",
+ "(A8,1)",
+ "(A9,2)");
+ }
+
+
+}
[08/19] DATAFU-27 Migrate build system to Gradle
Posted by wv...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/random/package-info.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/random/package-info.java b/src/java/datafu/pig/random/package-info.java
deleted file mode 100644
index 8c7750d..0000000
--- a/src/java/datafu/pig/random/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * UDFs dealing with randomness.
- */
-package datafu.pig.random;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/sampling/Reservoir.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/sampling/Reservoir.java b/src/java/datafu/pig/sampling/Reservoir.java
deleted file mode 100644
index 09b4c95..0000000
--- a/src/java/datafu/pig/sampling/Reservoir.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.sampling;
-
-import java.util.PriorityQueue;
-
-class Reservoir extends PriorityQueue<ScoredTuple>
-{
- private static final long serialVersionUID = 1L;
- private int numSamples;
-
- public Reservoir(int numSamples)
- {
- super(numSamples);
- this.numSamples = numSamples;
- }
-
- public boolean consider(ScoredTuple scoredTuple)
- {
- if (super.size() < numSamples) {
- return super.add(scoredTuple);
- } else {
- ScoredTuple head = super.peek();
- if (scoredTuple.score > head.score) {
- super.poll();
- return super.add(scoredTuple);
- }
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/sampling/ReservoirSample.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/sampling/ReservoirSample.java b/src/java/datafu/pig/sampling/ReservoirSample.java
deleted file mode 100644
index 48deaad..0000000
--- a/src/java/datafu/pig/sampling/ReservoirSample.java
+++ /dev/null
@@ -1,327 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.sampling;
-
-import java.io.IOException;
-
-import org.apache.pig.AccumulatorEvalFunc;
-import org.apache.pig.Algebraic;
-import org.apache.pig.EvalFunc;
-import org.apache.pig.builtin.Nondeterministic;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * Performs a simple random sample using an in-memory reservoir to produce
- * a uniformly random sample of a given size.
- *
- * <p>
- * This is similar to {@link SimpleRandomSample}, however it is guaranteed to produce
- * a sample of the given size. This comes at the cost of scalability.
- * {@link SimpleRandomSample} produces a sample of the desired size with likelihood of 99.99%,
- * while using less internal storage. ReservoirSample on the other hand uses internal storage
- * with size equaling the desired sample to guarantee the exact sample size.
- * </p>
- *
- * <p>
- * This algebraic implementation is backed by a heap and maintains the original roll in order
- * to compensate for skew.
- * </p>
- *
- * @author wvaughan
- *
- */
-@Nondeterministic
-public class ReservoirSample extends AccumulatorEvalFunc<DataBag> implements Algebraic
-{
- protected Integer numSamples;
-
- private Reservoir reservoir;
-
- protected ScoredTuple.ScoreGenerator scoreGen;
-
- private Reservoir getReservoir()
- {
- if (reservoir == null) {
- reservoir = new Reservoir(this.numSamples);
- }
- return reservoir;
- }
-
- public ReservoirSample(String numSamples)
- {
- this.numSamples = Integer.parseInt(numSamples);
- }
-
- protected ScoredTuple.ScoreGenerator getScoreGenerator()
- {
- if(this.scoreGen == null)
- {
- this.scoreGen = new ScoredTuple.PureRandomScoreGenerator();
- }
- return this.scoreGen;
- }
-
- @Override
- public void accumulate(Tuple input) throws IOException
- {
- DataBag samples = (DataBag) input.get(0);
- ScoredTuple.ScoreGenerator scoreGen = getScoreGenerator();
- for (Tuple sample : samples) {
- getReservoir().consider(new ScoredTuple(scoreGen.generateScore(sample), sample));
- }
- }
-
- @Override
- public void cleanup()
- {
- this.reservoir.clear();
- }
-
- @Override
- public DataBag getValue()
- {
- DataBag output = BagFactory.getInstance().newDefaultBag();
- for (ScoredTuple sample : getReservoir()) {
- output.add(sample.getTuple());
- }
- return output;
- }
-
- @Override
- public DataBag exec(Tuple input) throws IOException
- {
- DataBag samples = (DataBag)input.get(0);
- if (samples.size() <= numSamples) {
- return samples;
- }
- else
- {
- return super.exec(input);
- }
- }
-
- @Override
- public Schema outputSchema(Schema input) {
- try {
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.BAG) {
- throw new RuntimeException("Expected a BAG as input");
- }
-
- return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
- inputFieldSchema.schema, DataType.BAG));
- } catch (FrontendException e) {
- e.printStackTrace();
- throw new RuntimeException(e);
- }
- }
-
- String param = null;
- private String getParam()
- {
- if (param == null) {
- if (numSamples != null) {
- param = String.format("('%d')", numSamples);
- } else {
- param = "";
- }
- }
- return param;
- }
-
- @Override
- public String getInitial() {
- return Initial.class.getName()+getParam();
- }
-
- @Override
- public String getIntermed() {
- return Intermediate.class.getName()+getParam();
- }
-
- @Override
- public String getFinal() {
- return Final.class.getName()+getParam();
- }
-
- static public class Initial extends EvalFunc<Tuple>
- {
- int numSamples;
- private Reservoir reservoir;
- protected ScoredTuple.ScoreGenerator scoreGen;
- TupleFactory tupleFactory = TupleFactory.getInstance();
-
- public Initial(){}
-
- public Initial(String numSamples)
- {
- this.numSamples = Integer.parseInt(numSamples);
- }
-
- private Reservoir getReservoir()
- {
- if (reservoir == null) {
- reservoir = new Reservoir(this.numSamples);
- }
- return reservoir;
- }
-
- protected ScoredTuple.ScoreGenerator getScoreGenerator()
- {
- if(this.scoreGen == null)
- {
- this.scoreGen = new ScoredTuple.PureRandomScoreGenerator();
- }
- return this.scoreGen;
- }
-
- @Override
- public Tuple exec(Tuple input) throws IOException {
- DataBag output = BagFactory.getInstance().newDefaultBag();
-
- ScoredTuple.ScoreGenerator scoreGen = getScoreGenerator();
-
- DataBag samples = (DataBag) input.get(0);
- if (samples == null)
- {
- // do nothing
- }
- else if (samples.size() <= numSamples) {
- // no need to construct a reservoir, so just emit intermediate tuples
- for (Tuple sample : samples) {
- // add the score on to the intermediate tuple
- output.add(new ScoredTuple(scoreGen.generateScore(sample), sample).getIntermediateTuple(tupleFactory));
- }
- } else {
- getReservoir().clear();
-
- for (Tuple sample : samples) {
- getReservoir().consider(new ScoredTuple(scoreGen.generateScore(sample), sample));
- }
-
- for (ScoredTuple scoredTuple : getReservoir()) {
- // add the score on to the intermediate tuple
- output.add(scoredTuple.getIntermediateTuple(tupleFactory));
- }
- }
-
- return tupleFactory.newTuple(output);
- }
-
- }
-
- static public class Intermediate extends EvalFunc<Tuple>
- {
- int numSamples;
- private Reservoir reservoir;
- TupleFactory tupleFactory = TupleFactory.getInstance();
-
- public Intermediate(){}
-
- public Intermediate(String numSamples)
- {
- this.numSamples = Integer.parseInt(numSamples);
- }
-
- private Reservoir getReservoir()
- {
- if (reservoir == null) {
- reservoir = new Reservoir(this.numSamples);
- }
- return reservoir;
- }
-
- @Override
- public Tuple exec(Tuple input) throws IOException {
- getReservoir().clear();
-
- DataBag bagOfSamples = (DataBag) input.get(0);
- for (Tuple innerTuple : bagOfSamples) {
- DataBag samples = (DataBag) innerTuple.get(0);
-
- for (Tuple sample : samples) {
- // use the same score as previously generated
- getReservoir().consider(ScoredTuple.fromIntermediateTuple(sample));
- }
- }
-
- DataBag output = BagFactory.getInstance().newDefaultBag();
- for (ScoredTuple scoredTuple : getReservoir()) {
- // add the score on to the intermediate tuple
- output.add(scoredTuple.getIntermediateTuple(tupleFactory));
- }
-
- return tupleFactory.newTuple(output);
- }
-
- }
-
- static public class Final extends EvalFunc<DataBag>
- {
- int numSamples;
- private Reservoir reservoir;
- TupleFactory tupleFactory = TupleFactory.getInstance();
-
- public Final(){}
-
- public Final(String numSamples)
- {
- this.numSamples = Integer.parseInt(numSamples);
- }
-
- private Reservoir getReservoir()
- {
- if (reservoir == null) {
- reservoir = new Reservoir(this.numSamples);
- }
- return reservoir;
- }
-
- @Override
- public DataBag exec(Tuple input) throws IOException {
- getReservoir().clear();
-
- DataBag bagOfSamples = (DataBag) input.get(0);
- for (Tuple innerTuple : bagOfSamples) {
- DataBag samples = (DataBag) innerTuple.get(0);
-
- for (Tuple sample : samples) {
- // use the same score as previously generated
- getReservoir().consider(ScoredTuple.fromIntermediateTuple(sample));
- }
- }
-
- DataBag output = BagFactory.getInstance().newDefaultBag();
- for (ScoredTuple scoredTuple : getReservoir()) {
- // output the original tuple
- output.add(scoredTuple.getTuple());
- }
-
- return output;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/sampling/SampleByKey.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/sampling/SampleByKey.java b/src/java/datafu/pig/sampling/SampleByKey.java
deleted file mode 100644
index 90ea576..0000000
--- a/src/java/datafu/pig/sampling/SampleByKey.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.sampling;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.security.MessageDigest;
-import org.apache.pig.FilterFunc;
-import org.apache.pig.data.Tuple;
-
-/**
- * Provides a way of sampling tuples based on certain fields.
- * This is essentially equivalent to grouping on the fields, applying SAMPLE,
- * and then flattening. It is much more efficient though because it does not require
- * a reduce step.
- *
- * <p>
- * The method of sampling is to convert the key to a hash, derive a double value
- * from this, and then test this against a supplied probability. The double value
- * derived from a key is uniformly distributed between 0 and 1.
- * </p>
- *
- * <p>
- * The only required parameter is the sampling probability. This may be followed
- * by an optional seed value to control the random number generation.
- * </p>
- *
- * <p>
- * SampleByKey will work deterministically as long as the same seed is provided.
- * </p>
- *
- * <p>
- * Example:
- * <pre>
- * {@code
- * DEFINE SampleByKey datafu.pig.sampling.SampleByKey('0.5');
- *
- *-- input: (A,1), (A,2), (A,3), (B,1), (B,3)
- *
- * data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
- * output = FILTER data BY SampleByKey(A_id);
- *
- * --output: (B,1), (B,3)
- * }
- *
- * </pre>
- * </p>
- * @author evion
- *
- */
-
-public class SampleByKey extends FilterFunc
-{
- final static int PRIME_NUMBER = 31;
-
- Integer seed = null;
- double probability;
-
- public SampleByKey(String probability) {
- this.probability = Double.parseDouble(probability);
- }
-
- public SampleByKey(String probability, String salt) {
- this(probability);
- this.seed = salt.hashCode();
- }
-
- @Override
- public void setUDFContextSignature(String signature)
- {
- if (this.seed == null)
- this.seed = signature.hashCode();
- super.setUDFContextSignature(signature);
- }
-
- @Override
- public Boolean exec(Tuple input) throws IOException
- {
- int hashCode = 0;
- for(int i=0; i<input.size(); i++) {
- Object each = input.get(i);
- hashCode = hashCode*PRIME_NUMBER + each.hashCode();
- }
-
- try {
- return intToRandomDouble(hashCode) <= probability;
- }
- catch (Exception e) {
- e.printStackTrace();
- throw new RuntimeException("Exception on intToRandomDouble");
- }
- }
-
- private Double intToRandomDouble(int input) throws Exception
- {
- MessageDigest hasher = MessageDigest.getInstance("sha-1");
-
- ByteBuffer b = ByteBuffer.allocate(4+4);
- ByteBuffer b2 = ByteBuffer.allocate(20);
-
- b.putInt(seed);
- b.putInt(input);
- byte[] digest = hasher.digest(b.array());
- b.clear();
-
- b2.put(digest);
- b2.rewind();
- double result = (((double)b2.getInt())/Integer.MAX_VALUE + 1)/2;
- b2.clear();
-
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/sampling/ScoredTuple.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/sampling/ScoredTuple.java b/src/java/datafu/pig/sampling/ScoredTuple.java
deleted file mode 100644
index c793584..0000000
--- a/src/java/datafu/pig/sampling/ScoredTuple.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.sampling;
-
-import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-
-class ScoredTuple implements Comparable<ScoredTuple>
-{
- Double score;
- private Tuple tuple;
-
- public ScoredTuple()
- {
-
- }
-
- public ScoredTuple(Double score, Tuple tuple)
- {
- this.score = score;
- this.setTuple(tuple);
- }
-
- public Double getScore() {
- return score;
- }
-
- public void setScore(Double score) {
- this.score = score;
- }
-
- public Tuple getTuple() {
- return tuple;
- }
-
- public void setTuple(Tuple tuple) {
- this.tuple = tuple;
- }
-
- public Tuple getIntermediateTuple(TupleFactory tupleFactory)
- {
- Tuple intermediateTuple = tupleFactory.newTuple(2);
- try {
- intermediateTuple.set(0, score);
- intermediateTuple.set(1, tuple);
- }
- catch (ExecException e) {
- throw new RuntimeException(e);
- }
-
- return intermediateTuple;
- }
-
- public static ScoredTuple fromIntermediateTuple(Tuple intermediateTuple) throws ExecException
- {
- //Double score = ((Number)intermediateTuple.get(0)).doubleValue();
- try {
- Double score = (Double)intermediateTuple.get(0);
- Tuple originalTuple = (Tuple)intermediateTuple.get(1);
- return new ScoredTuple(score, originalTuple);
- } catch (Exception e) {
- throw new RuntimeException("Cannot deserialize intermediate tuple: "+intermediateTuple.toString(), e);
- }
- }
-
- @Override
- public int compareTo(ScoredTuple o) {
- if (score == null) {
- if (o == null) return 0;
- else return -1;
- }
- return score.compareTo(o.score);
- }
-
- static interface ScoreGenerator
- {
- double generateScore(Tuple sample) throws ExecException;
- }
-
- static class PureRandomScoreGenerator implements ScoreGenerator
- {
- public PureRandomScoreGenerator(){}
-
- public double generateScore(Tuple sample)
- {
- return Math.random();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/sampling/SimpleRandomSample.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/sampling/SimpleRandomSample.java b/src/java/datafu/pig/sampling/SimpleRandomSample.java
deleted file mode 100644
index 8e8debf..0000000
--- a/src/java/datafu/pig/sampling/SimpleRandomSample.java
+++ /dev/null
@@ -1,489 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.sampling;
-
-import java.io.IOException;
-import java.util.Comparator;
-
-import org.apache.commons.math.random.RandomDataImpl;
-import org.apache.pig.AlgebraicEvalFunc;
-import org.apache.pig.EvalFunc;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * Scalable simple random sampling (ScaSRS).
- * <p/>
- * This UDF implements a scalable simple random sampling algorithm described in
- *
- * <pre>
- * X. Meng, Scalable Simple Random Sampling and Stratified Sampling, ICML 2013.
- * </pre>
- *
- * It takes a bag of n items and a sampling probability p as the inputs, and outputs a
- * simple random sample of size exactly ceil(p*n) in a bag, with probability at least
- * 99.99%. For example, the following script generates a simple random sample with
- * sampling probability 0.1:
- *
- * <pre>
- * DEFINE SRS datafu.pig.sampling.SimpleRandomSample();
- *
- * item = LOAD 'input' AS (x:double);
- * sampled = FOREACH (GROUP item ALL) GENERATE FLATTEN(SRS(item, 0.01));
- * </pre>
- *
- * Optionally, user can provide a good lower bound of n as the third argument to help
- * reduce the size of intermediate data, for example:
- *
- * <pre>
- * DEFINE SRS datafu.pig.sampling.SimpleRandomSample();
- *
- * item = LOAD 'input' AS (x:double);
- * summary = FOREACH (GROUP item ALL) GENERATE COUNT(item) AS count;
- * sampled = FOREACH (GROUP item ALL) GENERATE FLATTEN(SRS(item, 0.01, summary.count));
- * </pre>
- *
- * This UDF is very useful for stratified sampling. For example, the following script
- * keeps all positive examples while downsampling negatives with probability 0.1:
- *
- * <pre>
- * DEFINE SRS datafu.pig.sampling.SimpleRandomSample();
- *
- * item = LOAD 'input' AS (x:double, label:int);
- * grouped = FOREACH (GROUP item BY label) GENERATE item, (group == 1 ? 1.0 : 0.1) AS p;
- * sampled = FOREACH grouped GENERATE FLATTEN(SRS(item, p));
- * </pre>
- *
- * In a Java Hadoop MapReduce job, we can output selected items directly using
- * MultipleOutputs. However, this feature is not available in a Pig UDF. So we still let
- * selected items go through the sort phase. However, as long as the sample size is not
- * huge, this should not be a big problem.
- *
- * In the first version, the sampling probability is specified in the constructor. This
- * method is deprecated now and will be removed in the next release.
- *
- * @author ximeng
- *
- */
-public class SimpleRandomSample extends AlgebraicEvalFunc<DataBag>
-{
- /**
- * Prefix for the output bag name.
- */
- public static final String OUTPUT_BAG_NAME_PREFIX = "SRS";
-
- private static final TupleFactory _TUPLE_FACTORY = TupleFactory.getInstance();
- private static final BagFactory _BAG_FACTORY = BagFactory.getInstance();
-
- public SimpleRandomSample()
- {
- // empty
- }
-
- /**
- * Constructs this UDF with a sampling probability.
- *
- * @deprecated Should specify the sampling probability in the function call.
- */
- @Deprecated
- public SimpleRandomSample(String samplingProbability)
- {
- double p = Double.parseDouble(samplingProbability);
- verifySamplingProbability(p);
- }
-
- @Override
- public String getInitial()
- {
- return Initial.class.getName();
- }
-
- @Override
- public String getIntermed()
- {
- return Intermediate.class.getName();
- }
-
- @Override
- public String getFinal()
- {
- return Final.class.getName();
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try
- {
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.BAG)
- {
- throw new RuntimeException("Expected a BAG as input");
- }
-
- return new Schema(new Schema.FieldSchema(super.getSchemaName(OUTPUT_BAG_NAME_PREFIX,
- input),
- inputFieldSchema.schema,
- DataType.BAG));
- }
- catch (FrontendException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- static public class Initial extends EvalFunc<Tuple>
- {
- // Should avoid creating many random number generator instances.
- private static RandomDataImpl _RNG = new RandomDataImpl();
-
- synchronized private static double nextDouble()
- {
- return _RNG.nextUniform(0.0d, 1.0d);
- }
-
- public Initial()
- {
- // empty
- }
-
- @Deprecated
- public Initial(String samplingProbability)
- {
- _p = Double.parseDouble(samplingProbability);
- }
-
- private boolean _first = true;
- private double _p = -1.0d; // the sampling probability
- private long _n1 = 0L; // the input lower bound of the size of the population
- private long _localCount = 0L; // number of items processed by this instance
-
- @Override
- public Tuple exec(Tuple input) throws IOException
- {
- int numArgs = input.size();
-
- // The first if clause is for backward compatibility, which should be removed
- // after we remove specifying sampling probability in the constructor.
- if(numArgs == 1)
- {
- if(_p < 0.0d)
- {
- throw new IllegalArgumentException("Sampling probability is not given.");
- }
- }
- else if (numArgs < 2 || numArgs > 3)
- {
- throw new IllegalArgumentException("The input tuple should have either two or three fields: "
- + "a bag of items, the sampling probability, "
- + "and optionally a good lower bound of the size of the population or the exact number.");
- }
-
- DataBag items = (DataBag) input.get(0);
- long numItems = items.size();
- _localCount += numItems;
-
- // This is also for backward compatibility. Should change to
- // double p = ((Number) input.get(1)).doubleValue();
- // after we remove specifying sampling probability in the constructor.
- double p = numArgs == 1 ? _p : ((Number) input.get(1)).doubleValue();
- if (_first)
- {
- _p = p;
- verifySamplingProbability(p);
- }
- else
- {
- if (p != _p)
- {
- throw new IllegalArgumentException("The sampling probability must be a scalar, but found two different values: "
- + _p + " and " + p + ".");
- }
- }
-
- long n1 = 0L;
- if (numArgs > 2)
- {
- n1 = ((Number) input.get(2)).longValue();
-
- if (_first)
- {
- _n1 = n1;
- }
- else
- {
- if (n1 != _n1)
- {
- throw new IllegalArgumentException("The lower bound of the population size must be a scalar, but found two different values: "
- + _n1 + " and " + n1 + ".");
- }
- }
- }
-
- _first = false;
-
- // Use the local count if the input lower bound is smaller.
- n1 = Math.max(n1, _localCount);
-
- DataBag selected = _BAG_FACTORY.newDefaultBag();
- DataBag waiting = _BAG_FACTORY.newDefaultBag();
-
- if (n1 > 0L)
- {
- double q1 = getQ1(n1, p);
- double q2 = getQ2(n1, p);
-
- for (Tuple t : items)
- {
- double x = nextDouble();
- if (x < q1)
- {
- selected.add(t);
- }
- else if (x < q2)
- {
- waiting.add(new ScoredTuple(x, t).getIntermediateTuple(_TUPLE_FACTORY));
- }
- }
- }
-
- /*
- * The output tuple contains the following fields: sampling probability (double),
- * number of processed items in this tuple (long), a good lower bound of the size of
- * the population or the exact number (long), a bag of selected items (bag), and a
- * bag of waitlisted items with scores (bag).
- */
- Tuple output = _TUPLE_FACTORY.newTuple();
-
- output.append(p);
- output.append(numItems);
- output.append(n1);
- output.append(selected);
- output.append(waiting);
-
- return output;
- }
- }
-
- public static class Intermediate extends EvalFunc<Tuple>
- {
- public Intermediate()
- {
- // empty
- }
-
- @Deprecated
- public Intermediate(String samplingProbability)
- {
- // empty
- }
-
- @Override
- public Tuple exec(Tuple input) throws IOException
- {
- DataBag bag = (DataBag) input.get(0);
-
- DataBag selected = _BAG_FACTORY.newDefaultBag();
- DataBag aggWaiting = _BAG_FACTORY.newDefaultBag();
-
- boolean first = true;
- double p = 0.0d;
- long numItems = 0L; // number of items processed, including rejected
- long n1 = 0L;
-
- for (Tuple tuple : bag)
- {
- if (first)
- {
- p = (Double) tuple.get(0);
- first = false;
- }
-
- numItems += (Long) tuple.get(1);
- n1 = Math.max((Long) tuple.get(2), numItems);
-
- selected.addAll((DataBag) tuple.get(3));
- aggWaiting.addAll((DataBag) tuple.get(4));
- }
-
- DataBag waiting = _BAG_FACTORY.newDefaultBag();
-
- if (n1 > 0L)
- {
- double q1 = getQ1(n1, p);
- double q2 = getQ2(n1, p);
-
- for (Tuple t : aggWaiting)
- {
- ScoredTuple scored = ScoredTuple.fromIntermediateTuple(t);
-
- if (scored.getScore() < q1)
- {
- selected.add(scored.getTuple());
- }
- else if (scored.getScore() < q2)
- {
- waiting.add(t);
- }
- }
- }
-
- Tuple output = _TUPLE_FACTORY.newTuple();
-
- output.append(p);
- output.append(numItems);
- output.append(n1);
- output.append(selected);
- output.append(waiting);
-
- return output;
- }
- }
-
- static public class Final extends EvalFunc<DataBag>
- {
- public Final()
- {
- // empty
- }
-
- @Deprecated
- public Final(String samplingProbability)
- {
- // empty
- }
-
- @Override
- public DataBag exec(Tuple input) throws IOException
- {
- DataBag bag = (DataBag) input.get(0);
-
- boolean first = true;
- double p = 0.0d; // the sampling probability
- long n = 0L; // the size of the population (total number of items)
-
- DataBag selected = _BAG_FACTORY.newDefaultBag();
- DataBag waiting = _BAG_FACTORY.newSortedBag(ScoredTupleComparator.getInstance());
-
- for (Tuple tuple : bag)
- {
- if (first)
- {
- p = (Double) tuple.get(0);
- first = false;
- }
-
- n += (Long) tuple.get(1);
- selected.addAll((DataBag) tuple.get(3));
- waiting.addAll((DataBag) tuple.get(4));
- }
-
- long numSelected = selected.size();
- long numWaiting = waiting.size();
-
- long s = (long) Math.ceil(p * n); // sample size
-
- System.out.println("To sample " + s + " items from " + n + ", we pre-selected "
- + numSelected + ", and waitlisted " + waiting.size() + ".");
-
- long numNeeded = s - selected.size();
-
- if (numNeeded < 0)
- {
- System.err.println("Pre-selected " + numSelected + " items, but only needed " + s
- + ".");
- }
-
- for (Tuple scored : waiting)
- {
- if (numNeeded <= 0)
- {
- break;
- }
- selected.add(ScoredTuple.fromIntermediateTuple(scored).getTuple());
- numNeeded--;
- }
-
- if (numNeeded > 0)
- {
- System.err.println("The waiting list only has " + numWaiting
- + " items, but needed " + numNeeded + " more.");
- }
-
- return selected;
- }
- }
-
- // computes a threshold to select items
- private static double getQ1(long n, double p)
- {
- double t1 = 20.0d / (3.0d * n);
- double q1 = p + t1 - Math.sqrt(t1 * t1 + 3.0d * t1 * p);
- return q1;
- }
-
- // computes a threshold to reject items
- private static double getQ2(long n, double p)
- {
- double t2 = 10.0d / n;
- double q2 = p + t2 + Math.sqrt(t2 * t2 + 2.0d * t2 * p);
- return q2;
- }
-
- private static void verifySamplingProbability(double p)
- {
- if(p < 0.0 || p > 1.0)
- {
- throw new IllegalArgumentException("Sampling probabiilty must be inside [0, 1].");
- }
- }
-
- static class ScoredTupleComparator implements Comparator<Tuple>
- {
- public static final ScoredTupleComparator getInstance()
- {
- return _instance;
- }
-
- private static final ScoredTupleComparator _instance = new ScoredTupleComparator();
-
- @Override
- public int compare(Tuple o1, Tuple o2)
- {
- try
- {
- ScoredTuple t1 = ScoredTuple.fromIntermediateTuple(o1);
- ScoredTuple t2 = ScoredTuple.fromIntermediateTuple(o2);
- return t1.getScore().compareTo(t2.getScore());
- }
- catch (Throwable e)
- {
- throw new RuntimeException("Cannot compare " + o1 + " and " + o2 + ".", e);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/sampling/SimpleRandomSampleWithReplacementElect.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/sampling/SimpleRandomSampleWithReplacementElect.java b/src/java/datafu/pig/sampling/SimpleRandomSampleWithReplacementElect.java
deleted file mode 100644
index a59816a..0000000
--- a/src/java/datafu/pig/sampling/SimpleRandomSampleWithReplacementElect.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.sampling;
-
-import java.io.IOException;
-import java.util.Comparator;
-
-import org.apache.pig.AlgebraicEvalFunc;
-import org.apache.pig.EvalFunc;
-import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * Select the candidate with the smallest score for each position from the candidates
- * proposed by {@link SimpleRandomSampleWithReplacementVote}.
- *
- * @see SimpleRandomSampleWithReplacementVote
- *
- * @author ximeng
- *
- */
-public class SimpleRandomSampleWithReplacementElect extends AlgebraicEvalFunc<DataBag>
-{
- /**
- * Prefix for the output bag name.
- */
- public static final String OUTPUT_BAG_NAME_PREFIX = "SRSWR_ELECT";
-
- public static final TupleFactory tupleFactory = TupleFactory.getInstance();
- public static final BagFactory bagFactory = BagFactory.getInstance();
-
- static class CandidateComparator implements Comparator<Tuple>
- {
- private static CandidateComparator _instance = new CandidateComparator();
-
- public static CandidateComparator get()
- {
- return _instance;
- }
-
- private CandidateComparator()
- {
- // singleton
- }
-
- @Override
- public int compare(Tuple o1, Tuple o2)
- {
- try
- {
- // first by position
- int c1 = ((Integer) o1.get(0)).compareTo((Integer) o2.get(0));
- if (c1 != 0)
- {
- return c1;
- }
- else
- {
- // then by score
- return ((Double) o1.get(1)).compareTo((Double) o2.get(1));
- }
- }
- catch (ExecException e)
- {
- throw new RuntimeException("Error comparing tuples " + o1 + " and " + o2, e);
- }
- }
- }
-
- @Override
- public String getInitial()
- {
- return Initial.class.getName();
- }
-
- @Override
- public String getIntermed()
- {
- return Intermediate.class.getName();
- }
-
- @Override
- public String getFinal()
- {
- return Final.class.getName();
- }
-
- static public class Initial extends EvalFunc<Tuple>
- {
- @Override
- public Tuple exec(Tuple input) throws IOException
- {
- // output each input candidate
- return input;
- }
- }
-
- static public class Intermediate extends EvalFunc<Tuple>
- {
- @Override
- public Tuple exec(Tuple tuple) throws IOException
- {
- // sort candidates first by index, then by key
- DataBag candidates = bagFactory.newSortedBag(CandidateComparator.get());
- for (Tuple intermediateOutputTuple : (DataBag) tuple.get(0))
- {
- candidates.addAll((DataBag) intermediateOutputTuple.get(0));
- }
-
- DataBag outputBag = bagFactory.newDefaultBag();
- int i = -1;
- for (Tuple candidate : candidates)
- {
- int pos = (Integer) candidate.get(0);
- if (pos > i)
- {
- outputBag.add(candidate);
- i = pos;
- }
- }
-
- return tupleFactory.newTuple(outputBag);
- }
-
- }
-
- static public class Final extends EvalFunc<DataBag>
- {
- @Override
- public DataBag exec(Tuple tuple) throws IOException
- {
- DataBag candidates = bagFactory.newSortedBag(CandidateComparator.get());
- for (Tuple intermediateOutputTuple : (DataBag) tuple.get(0))
- {
- candidates.addAll((DataBag) intermediateOutputTuple.get(0));
- }
-
- DataBag outputBag = bagFactory.newDefaultBag();
- int i = -1;
- for (Tuple candidate : candidates)
- {
- int pos = (Integer) candidate.get(0);
- if (pos > i)
- {
- outputBag.add((Tuple) candidate.get(2));
- i = pos;
- }
- }
- return outputBag;
- }
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try
- {
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.BAG)
- {
- throw new RuntimeException("Expected a BAG as input");
- }
-
- // the output is a bag of selected items
- Schema outputSchema =
- new Schema(new Schema.FieldSchema(super.getSchemaName(OUTPUT_BAG_NAME_PREFIX,
- input),
- inputFieldSchema.schema.getField(0).schema.getField(2).schema,
- DataType.BAG));
-
- return outputSchema;
- }
- catch (FrontendException e)
- {
- throw new RuntimeException("Error deriving output schema.", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/sampling/SimpleRandomSampleWithReplacementVote.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/sampling/SimpleRandomSampleWithReplacementVote.java b/src/java/datafu/pig/sampling/SimpleRandomSampleWithReplacementVote.java
deleted file mode 100644
index 598e58c..0000000
--- a/src/java/datafu/pig/sampling/SimpleRandomSampleWithReplacementVote.java
+++ /dev/null
@@ -1,281 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.sampling;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.math.MathException;
-import org.apache.commons.math.random.RandomDataImpl;
-import org.apache.pig.EvalFunc;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import com.google.common.primitives.Ints;
-
-/**
- * Scalable simple random sampling with replacement (ScaSRSWR).
- * <p/>
- * This UDF together with {@link SimpleRandomSampleWithReplacementElect} implement a
- * scalable algorithm for simple random sampling with replacement (SRSWR), which is a
- * randomized algorithm with a failure rate less than {@value #FAILURE_RATE}.
- * <p/>
- * Let s be the desired sample size. To compute an SRSWR sample of size s, for each output
- * position in {0, 1, ..., s-1}, we want to select an item from the population uniformly
- * at random. This algorithm consists of two stages: vote and election. In the vote stage,
- * this UDF {@link SimpleRandomSampleWithReplacementVote} votes items, called candidates,
- * for each position. In the election stage, the paired UDF
- * {@link SimpleRandomSampleWithReplacementElect} elects one candidate for each position.
- * The algorithm succeeds if we have at least one candidate for each position.
- * <p/>
- * To use this UDF pair, user needs to provide: 1) the desired sample size, 2) a good
- * lower bound of the population size or the exact size. The input to the vote UDF
- * {@link SimpleRandomSampleWithReplacementVote} is a tuple that consists of a bag of
- * items, the desired sample size (int), and the population size (long) or a good lower
- * bound of it, where the latter two must be scalars. The output from the vote UDF is a
- * tuple that consists of position:int, score:double, and candidate. The input to the
- * elect UDF {@link SimpleRandomSampleWithReplacementElect} is a tuple that contains all
- * candidates voted by the vote UDF for some positions. The output from the elect UDF is a
- * bag of sampled items.
- * <p/>
- * For example, the following script generates a sample of size 100000 with replacement:
- *
- * <pre>
- * DEFINE SRSWR_VOTE datafu.pig.sampling.SimpleRandomSampleWithReplacementVote();
- * DEFINE SRSWR_ELECT datafu.pig.sampling.SimpleRandomSampleWithReplacementElect();
- *
- * item = LOAD 'input' AS (x:double);
- * summary = FOREACH (GROUP item ALL) GENERATE COUNT(item) AS count;
- * candidates = FOREACH item GENERATE FLATTEN(SRSWR_VOTE(TOBAG(x), 100000, summary.count));
- * sampled = FOREACH (GROUP candidates BY position PARALLEL 10) GENERATE FLATTEN(SRSWR_ELECT(candidates));
- * </pre>
- *
- * Because for election we only need to group candidates voted for the same position, this
- * algorithm can use many reducers to consume the candidates. See the "PARALLEL 10"
- * statement above. If the item to sample is the entire row, use TOBAG(TOTUPLE(*)).
- * <p/>
- * SRSWR is heavily used in bootstrapping. Bootstrapping can be done easily with this UDF
- * pair. For example, the following script generates 100 bootstrap samples, computes the
- * mean value for each sample, and then outputs the bootstrap estimates.
- *
- * <pre>
- * summary = FOREACH (GROUP item ALL) GENERATE AVG(item.x) AS mean, COUNT(item) AS count;
- * candidates = FOREACH item GENERATE FLATTEN(SRSWR_VOTE(TOBAG(x), summary.count*100, summary.count));
- * sampled = FOREACH (GROUP candidates BY (position % 100) PARALLEL 10) GENERATE AVG(SRSWR_ELECT(candidates)) AS mean;
- * bootstrap = FOREACH (GROUP sampled ALL) GENERATE summary.mean AS mean, sampled.mean AS bootstrapMeans;
- * </pre>
- *
- * Another usage of this UDF pair is to generate random pairs or tuples without computing
- * the cross product, where each pair or tuple consist of items from different input
- * sources. Let s be the number of random tuples we want to generate. For each input
- * source, simply use the vote UDF to propose candidates, then join the candidates from
- * different sources by their positions and for each position use the elect UDF to select
- * one candidate from each source to form the pair or tuple for that position.
- * <p/>
- * The algorithm is a simple extension to the work
- *
- * <pre>
- * X. Meng, Scalable Simple Random Sampling and Stratified Sampling, ICML 2013.
- * </pre>
- *
- * Basically, for each output position, it performs a random sort on the population
- * (associates each item with a random score independently drawn from the uniform
- * distribution and then sorts items based on the scores), and picks the one that has the
- * smallest score. However, a probabilistic threshold is used to avoid sorting the entire
- * population. For example, if the population size is one billion and the random score
- * generated for an item is 0.9, very likely it won't become the smallest and hence we do
- * not need to propose it as a candidate.
- * <p/>
- * More precisely, let n be the population size, n1 be a good lower bound of n, s be the
- * sample size, delta be the failure rate, and q be the threshold. For each output
- * position the probability of all random scores being greater than q is (1-q)^n. Thus, if
- * we throw away items with associated scores greater than q, with probability at least 1
- * - s*(1-q)^n, we can still capture the item with the smallest score for each position.
- * Fix delta = s*(1-q)^n and solve for q, we get q = 1-exp(log(delta/s)/n), Note that
- * replacing n by n1 < n can only decrease the failure rate, though at the cost of
- * increased number of candidates. The expected number of candidates is (1 -
- * exp(log(delta/s)/n1)*s*n. When n1 equals n, this number is approximately
- * s*log(s/delta).
- * <p/>
- * Generating a random score for each (item, position) pair is very expensive and
- * unnecessary. For each item, the number of positions for which it gets voted follows a
- * binomial distribution B(s,q). We can simply draw a number from this distribution,
- * determine the positions by sampling without replacement, and then generate random
- * scores for those positions. This reduces the running time significantly.
- * <p/>
- * Since for each position we only need the candidate with the smallest score, we
- * implement a combiner to reduce the size of intermediate data in the elect UDF
- * {@link SimpleRandomSampleWithReplacementElect}.
- *
- * @see SimpleRandomSampleWithReplacementElect
- * @see <a href="http://en.wikipedia.org/wiki/Bootstrapping_(statistics) target="_blank
- * ">Boostrapping (Wikipedia)</a>
- *
- * @author ximeng
- *
- */
-public class SimpleRandomSampleWithReplacementVote extends EvalFunc<DataBag>
-{
- public static final String OUTPUT_BAG_NAME_PREFIX = "SRSWR_VOTE";
- public static final String CANDIDATE_FIELD_NAME = "candidate";
- public static final String POSITION_FIELD_NAME = "position";
- public static final String SCORE_FIELD_NAME = "score";
- public static final double FAILURE_RATE = 1e-4;
-
- private static final TupleFactory tupleFactory = TupleFactory.getInstance();
- private static final BagFactory bagFactory = BagFactory.getInstance();
-
- private RandomDataImpl _rdg = new RandomDataImpl();
-
- /**
- * Samples k integers from [0, n) without replacement efficiently.
- *
- * If k is small, we can repeatedly draw integers from [0, n) until there are k distinct
- * values. For each trial, with probability at least (n-k)/n, we can draw a new value.
- * So the expected number of trials is smaller than (k*n)/(n-k), which is a very rough
- * bound. If k is large, we use the selection-rejection sampling algorithm. Basically,
- * we want the running time to be O(k).
- *
- */
- private int[] sampleWithoutReplacement(int n, int k)
- {
- if (k == 0)
- {
- return new int[] {};
- }
-
- if (k < n / 3L)
- {
- Set<Integer> sample = Sets.newHashSetWithExpectedSize(k);
-
- // The expected number of iterations is less than 1.5*k
- while (sample.size() < k)
- {
- sample.add(_rdg.nextInt(0, n - 1));
- }
-
- return Ints.toArray(sample);
- }
- else
- {
- int[] sample = new int[k];
-
- int i = 0;
- for (int j = 0; j < n && i < k; ++j)
- {
- if (_rdg.nextUniform(0.0d, 1.0d) < 1.0d * (k - i) / (n - j))
- {
- sample[i] = j;
- i++;
- }
- }
-
- return sample;
- }
- }
-
- @Override
- public DataBag exec(Tuple tuple) throws IOException
- {
- if (tuple.size() != 3)
- {
- throw new IllegalArgumentException("The input arguments are: "
- + "a bag of items, the desired sample size (int), and the population size (long) or a good lower bound of it");
- }
-
- DataBag items = (DataBag) tuple.get(0);
- int sampleSize = ((Number) tuple.get(1)).intValue();
- long count = ((Number) tuple.get(2)).longValue();
-
- /*
- * The following threshold is to guarantee that each output position contains at least
- * one candidate with high probability.
- */
- double threshold = 1.0d - Math.exp(Math.log(FAILURE_RATE / sampleSize) / count);
-
- DataBag candidates = bagFactory.newDefaultBag();
-
- for (Tuple item : items)
- {
- // Should be able to support long sample size if nextBinomial supports long.
- int numOutputPositions;
- try
- {
- numOutputPositions = _rdg.nextBinomial(sampleSize, threshold);
- }
- catch (MathException e)
- {
- throw new RuntimeException("Failed to generate a binomial value with n = "
- + sampleSize + " and p = " + threshold, e);
- }
- for (int outputPosition : sampleWithoutReplacement(sampleSize, numOutputPositions))
- {
- Tuple candidate = tupleFactory.newTuple();
- candidate.append(outputPosition);
- candidate.append(_rdg.nextUniform(0.0d, 1.0d)); // generate a random score
- candidate.append(item);
- candidates.add(candidate);
- }
- }
-
- return candidates;
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try
- {
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.BAG)
- {
- throw new RuntimeException("Expected a BAG as input");
- }
-
- List<Schema.FieldSchema> fieldSchemas = Lists.newArrayList();
-
- fieldSchemas.add(new Schema.FieldSchema(POSITION_FIELD_NAME, DataType.INTEGER));
- fieldSchemas.add(new Schema.FieldSchema(SCORE_FIELD_NAME, DataType.DOUBLE));
- fieldSchemas.add(new Schema.FieldSchema(CANDIDATE_FIELD_NAME,
- inputFieldSchema.schema.getField(0).schema));
-
- Schema outputSchema =
- new Schema(new Schema.FieldSchema(super.getSchemaName(OUTPUT_BAG_NAME_PREFIX,
- input),
- new Schema(fieldSchemas),
- DataType.BAG));
-
- return outputSchema;
- }
- catch (FrontendException e)
- {
- throw new RuntimeException("Error deriving output schema.", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/sampling/WeightedReservoirSample.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/sampling/WeightedReservoirSample.java b/src/java/datafu/pig/sampling/WeightedReservoirSample.java
deleted file mode 100644
index 92af6a3..0000000
--- a/src/java/datafu/pig/sampling/WeightedReservoirSample.java
+++ /dev/null
@@ -1,273 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.sampling;
-
-import java.util.List;
-
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.builtin.Nondeterministic;
-import org.apache.pig.backend.executionengine.ExecException;
-
-/**
- * <p>
- * Performs a weighted random sample using an in-memory reservoir to produce
- * a weighted random sample of a given size based on the A-Res algorithm described in
- * {@link <a href="http://utopia.duth.gr/~pefraimi/research/data/2007EncOfAlg.pdf" target="_blank">paper</a>}.
- * </p>
- * <p>
- * Species with larger weight have higher probability to be selected in the final sample set.
- * </p>
- * <p>
- * This UDF inherits from {@link ReservoirSample} and it is guaranteed to produce
- * a sample of the given size. Similarly it comes at the cost of scalability.
- * since it uses internal storage with size equaling the desired sample to guarantee the exact sample size.
- * </p>
- * <p>
- * Its constructor takes 2 arguments.
- * <ul>
- * <li>The 1st argument specifies the sample size which should be a string of positive integer.
- * <li>The 2nd argument specifies the index of the weight field in the input tuple,
- * which should be a string of non-negative integer that is no greater than the input tuple size.
- * </ul>
- * </p>
- * <p>
- * Example:
- * <pre>
- * {@code
- * define WeightedSample datafu.pig.sampling.WeightedReservoirSample('1','1');
- * input = LOAD 'input' AS (v1:chararray, v2:INT);
- * input_g = GROUP input ALL;
- * sampled = FOREACH input_g GENERATE WeightedSample(input);
- * }
- * </pre>
- * </p>
- * @author wjian
- */
-
-@Nondeterministic
-public class WeightedReservoirSample extends ReservoirSample {
-
- private Integer weightIdx;
-
- public WeightedReservoirSample(String strNumSamples, String strWeightIdx)
- {
- super(strNumSamples);
- this.weightIdx = Integer.parseInt(strWeightIdx);
- if(this.weightIdx < 0) {
- throw new IllegalArgumentException("Invalid negative index of weight field argument for WeightedReserviorSample constructor: "
- + strWeightIdx);
- }
- }
-
- @Override
- protected ScoredTuple.ScoreGenerator getScoreGenerator()
- {
- if(super.scoreGen == null)
- {
- super.scoreGen = new InverseWeightScoreGenerator(this.weightIdx);
- }
- return this.scoreGen;
- }
-
- @Override
- public Schema outputSchema(Schema input) {
- try {
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.BAG) {
- throw new RuntimeException("Expected a BAG as input");
- }
-
- Schema inputBagSchema = inputFieldSchema.schema;
-
- if (inputBagSchema.getField(0).type != DataType.TUPLE)
- {
- throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
- DataType.findTypeName(inputBagSchema.getField(0).type)));
- }
-
- Schema tupleSchema = inputBagSchema.getField(0).schema;
-
- if(tupleSchema == null) {
- throw new RuntimeException("The tuple of input bag has no schema");
- }
-
- List<Schema.FieldSchema> fieldSchemaList = tupleSchema.getFields();
-
- if(fieldSchemaList == null || fieldSchemaList.size() <= Math.max(0, this.weightIdx)) {
- throw new RuntimeException("The field schema of the input tuple is null " +
- "or the tuple size is no more than the weight field index: "
- + this.weightIdx);
- }
-
- if(fieldSchemaList.get(this.weightIdx).type != DataType.INTEGER &&
- fieldSchemaList.get(this.weightIdx).type != DataType.LONG &&
- fieldSchemaList.get(this.weightIdx).type != DataType.FLOAT &&
- fieldSchemaList.get(this.weightIdx).type != DataType.DOUBLE)
- {
- String[] expectedTypes = new String[] {DataType.findTypeName(DataType.INTEGER),
- DataType.findTypeName(DataType.LONG),
- DataType.findTypeName(DataType.FLOAT),
- DataType.findTypeName(DataType.DOUBLE)};
- throw new RuntimeException("Expect the type of the weight field of the input tuple to be of (" +
- java.util.Arrays.toString(expectedTypes) + "), but instead found (" +
- DataType.findTypeName(fieldSchemaList.get(this.weightIdx).type) + "), weight field: " +
- this.weightIdx);
- }
-
- return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
- inputFieldSchema.schema, DataType.BAG));
- } catch (FrontendException e) {
- e.printStackTrace();
- throw new RuntimeException(e);
- }
- }
-
- String param = null;
-
- private String getParam()
- {
- if (this.param == null) {
- if(super.numSamples != null && this.weightIdx != null) {
- this.param = String.format("('%d','%d')",
- super.numSamples,
- this.weightIdx);
- } else {
- this.param = "";
- }
- }
-
- return this.param;
- }
-
-
- @Override
- public String getInitial()
- {
- return Initial.class.getName() + getParam();
- }
-
- @Override
- public String getIntermed()
- {
- return Intermediate.class.getName() + getParam();
- }
-
- @Override
- public String getFinal()
- {
- return Final.class.getName() + getParam();
- }
-
- static public class Initial extends ReservoirSample.Initial
- {
- private Integer weightIdx;
-
- public Initial()
- {
- super();
- this.weightIdx = null;
- }
-
- public Initial(String strNumSamples, String strWeightIdx)
- {
- super(strNumSamples);
- this.weightIdx = Integer.parseInt(strWeightIdx);
- if(this.weightIdx < 0) {
- throw new IllegalArgumentException("Invalid negative index of weight field for WeightedReserviorSample.Initial constructor: "
- + strWeightIdx);
- }
- }
-
- @Override
- protected ScoredTuple.ScoreGenerator getScoreGenerator()
- {
- if(super.scoreGen == null)
- {
- super.scoreGen = new InverseWeightScoreGenerator(this.weightIdx);
- }
- return super.scoreGen;
- }
- }
-
- static public class Intermediate extends ReservoirSample.Intermediate
- {
- public Intermediate()
- {
- super();
- }
-
- public Intermediate(String strNumSamples, String strWeightIdx)
- {
- super(strNumSamples);
- }
- }
-
- static public class Final extends ReservoirSample.Final
- {
- public Final()
- {
- super();
- }
-
- public Final(String strNumSamples, String strWeightIdx)
- {
- super(strNumSamples);
- }
- }
-
- static class InverseWeightScoreGenerator implements ScoredTuple.ScoreGenerator
- {
- //index of the weight field of the input tuple
- private int weightIdx;
-
- InverseWeightScoreGenerator(Integer weightIdx)
- {
- if(weightIdx == null || weightIdx < 0) {
- throw new IllegalArgumentException("Invalid null or negative weight index input: " + weightIdx);
- }
- this.weightIdx = weightIdx;
- }
-
- @Override
- public double generateScore(Tuple sample) throws ExecException
- {
- if(this.weightIdx >= sample.size())
- {
- throw new ExecException(String.format("Weight index %d is outside tuple bounds", this.weightIdx));
- }
- if (sample.get(this.weightIdx) == null)
- {
- throw new ExecException(String.format("null value for weight at index %d",this.weightIdx));
- }
- double weight = ((Number)sample.get(this.weightIdx)).doubleValue();
- if(Double.compare(weight, 0.0) <= 0)
- {
- //non-positive weight should be avoided
- throw new ExecException(String.format("Invalid sample weight [%f]. It should be a positive real number", weight));
- }
- //a differnt approach to try: u^(1/w) could be exp(log(u)/w) ?
- return Math.pow(Math.random(), 1/weight);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/sampling/WeightedSample.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/sampling/WeightedSample.java b/src/java/datafu/pig/sampling/WeightedSample.java
deleted file mode 100644
index a77164d..0000000
--- a/src/java/datafu/pig/sampling/WeightedSample.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.sampling;
-
-import java.io.IOException;
-import java.util.Random;
-
-import org.apache.pig.EvalFunc;
-import org.apache.pig.builtin.Nondeterministic;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * Performs weighted bernoulli sampling on a bag.
- *
- * <p>
- * Create a new bag by performing a weighted sampling without replacement
- * from the input bag. Sampling is biased according to a weight that
- * is part of the inner tuples in the bag. That is, tuples with relatively
- * high weights are more likely to be chosen over tuples with low weights.
- * Optionally, a limit on the number of items to return may be specified.
- * </p>
- *
- * <p>
- * Example:
- * <pre>
- * {@code
- * define WeightedSample datafu.pig.sampling.WeightedSample()
- *
- * -- input:
- * -- ({(a, 100),(b, 1),(c, 5),(d, 2)})
- * input = LOAD 'input' AS (A: bag{T: tuple(name:chararray,score:int)});
- *
- * output1 = FOREACH input GENERATE WeightedSample(A,1);
- * -- output1:
- * -- uses the field indexed by 1 as a score
- * -- ({(a,100),(c,5),(b,1),(d,2)}) -- example of random
- *
- * -- sample using the second column (index 1) and keep only the top 3
- * output2 = FOREACH input GENERATE WeightedSample(A,1,3);
- * -- output2:
- * -- ({(a,100),(c,5),(b,1)})
- * }
- * </pre>
- */
-@Nondeterministic
-public class WeightedSample extends EvalFunc<DataBag>
-{
- BagFactory bagFactory = BagFactory.getInstance();
- Long seed = null;
-
- public WeightedSample() {
- }
-
- public WeightedSample(String seed) {
- this.seed = Long.parseLong(seed);
- }
-
- @Override
- public DataBag exec(Tuple input) throws IOException {
- DataBag output = bagFactory.newDefaultBag();
-
- DataBag samples = (DataBag) input.get(0);
- if (samples == null || samples.size() == 0) {
- return output; // if we are given null we will return an empty bag
- }
- int numSamples = (int) samples.size();
- if (numSamples == 1) return samples;
-
- Tuple[] tuples = new Tuple[numSamples];
- int tupleIndex = 0;
- for (Tuple tuple : samples) {
- tuples[tupleIndex] = tuple;
- tupleIndex++;
- }
-
- double[] scores = new double[numSamples];
- int scoreIndex = ((Number)input.get(1)).intValue();
- tupleIndex = 0;
- for (Tuple tuple : samples) {
- double score = ((Number)tuple.get(scoreIndex)).doubleValue();
- score = Math.max(score, Double.MIN_NORMAL); // negative scores cause problems
- scores[tupleIndex] = score;
- tupleIndex++;
- }
-
- // accept any type of number for sample size, but convert to int
- int limitSamples = numSamples;
- if (input.size() == 3) {
- // sample limit included
- limitSamples = Math.min(((Number)input.get(2)).intValue(), numSamples);
- }
-
- /*
- * Here's how the algorithm works:
- *
- * 1. Create a cumulative distribution of the scores 2. Draw a random number 3. Find
- * the interval in which the drawn number falls into 4. Select the element
- * encompassing that interval 5. Remove the selected element from consideration 6.
- * Repeat 1-5 k times
- *
- * However, rather than removing the element (#5), which is expensive for an array,
- * this function performs some extra bookkeeping by replacing the selected element
- * with an element from the front of the array and truncating the front. This
- * complicates matters as the element positions have changed, so another mapping for
- * positions is needed.
- *
- * This is an O(k*n) algorithm, where k is the number of elements to sample and n is
- * the number of scores.
- */
- Random rng = null;
- if (seed == null) {
- rng = new Random();
- } else {
- rng = new Random(seed);
- }
-
- for (int k = 0; k < limitSamples; k++) {
- double val = rng.nextDouble();
- int idx = find_cumsum_interval(scores, val, k, numSamples);
- if (idx == numSamples)
- idx = rng.nextInt(numSamples - k) + k;
-
- output.add(tuples[idx]);
-
- scores[idx] = scores[k];
- tuples[idx] = tuples[k];
- }
-
- return output;
- }
-
- public int find_cumsum_interval(double[] scores, double val, int begin, int end) {
- double sum = 0.0;
- double cumsum = 0.0;
- for (int i = begin; i < end; i++) {
- sum += scores[i];
- }
-
- for (int i = begin; i < end; i++) {
- cumsum += scores[i];
- if ((cumsum / sum) > val)
- return i;
- }
- return end;
- }
-
- @Override
- public Schema outputSchema(Schema input) {
- try {
- if (!(input.size() == 2 || input.size() == 3))
- {
- throw new RuntimeException("Expected input to have two or three fields");
- }
-
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.BAG) {
- throw new RuntimeException("Expected a BAG as first input, got: "+inputFieldSchema.type);
- }
-
- if (input.getField(1).type != DataType.INTEGER) {
- throw new RuntimeException("Expected an INT as second input, got: "+input.getField(1).type);
- }
-
- if (input.size() == 3 && !(input.getField(2).type == DataType.INTEGER || input.getField(2).type == DataType.LONG)) {
- throw new RuntimeException("Expected an INT or LONG as second input, got: "+input.getField(2).type);
- }
-
- return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
- inputFieldSchema.schema, DataType.BAG));
- } catch (FrontendException e) {
- e.printStackTrace();
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/sampling/package-info.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/sampling/package-info.java b/src/java/datafu/pig/sampling/package-info.java
deleted file mode 100644
index 0c7d8bd..0000000
--- a/src/java/datafu/pig/sampling/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Sampling UDFs, including weighted sample, reservoir sampling, sampling by key, etc.
- */
-package datafu.pig.sampling;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/sessions/SessionCount.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/sessions/SessionCount.java b/src/java/datafu/pig/sessions/SessionCount.java
deleted file mode 100644
index fe68888..0000000
--- a/src/java/datafu/pig/sessions/SessionCount.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.sessions;
-
-import java.io.IOException;
-
-import org.apache.pig.Accumulator;
-import org.apache.pig.AccumulatorEvalFunc;
-import org.apache.pig.EvalFunc;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.Tuple;
-import org.joda.time.DateTime;
-import org.joda.time.Period;
-
-import datafu.pig.util.SimpleEvalFunc;
-
-/**
- * Performs a count of events, ignoring events which occur within the
- * same time window.
- * <p>
- * This is useful for tasks such as counting the number of page views per user since it:
- * a) prevent reloads and go-backs from overcounting actual views
- * b) captures the notion that views across multiple sessions are more meaningful
- * <p>
- * Input <b>must</b> be sorted ascendingly by time for this UDF to work.
- * <p>
- * Example:
- * <pre>
- * {@code
- *
- * %declare TIME_WINDOW 10m
- *
- * define SessionCount datafu.pig.sessions.SessionCount('$TIME_WINDOW');
- *
- * views = LOAD 'views' as (user_id:int, page_id:int, time:chararray);
- * views_grouped = GROUP views by (user_id, page_id);
- * view_counts = FOREACH views_grouped {
- * views = order views by time;
- * generate group.user_id as user_id,
- * group.page_id as page_id,
- * SessionCount(views.(time)) as count; }
- * }
- * </pre>
- *
- */
-public class SessionCount extends AccumulatorEvalFunc<Long>
-{
- private final long millis;
- private DateTime last_date;
- private long sum;
-
- public SessionCount(String timeSpec)
- {
- Period p = new Period("PT" + timeSpec.toUpperCase());
- this.millis = p.toStandardSeconds().getSeconds() * 1000;
- cleanup();
- }
-
- @Override
- public void accumulate(Tuple input) throws IOException
- {
- for (Tuple t : (DataBag) input.get(0)) {
- DateTime date = new DateTime(t.get(0));
-
- if (last_date == null) {
- last_date = date;
- sum = 1;
- } else if (date.isAfter(last_date.plus(this.millis)))
- sum += 1;
- else if (date.isBefore(last_date))
- throw new IOException("input time series is not sorted");
-
- last_date = date;
- }
- }
-
- @Override
- public Long getValue()
- {
- return sum;
- }
-
- @Override
- public void cleanup()
- {
- this.last_date = null;
- this.sum = 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/sessions/Sessionize.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/sessions/Sessionize.java b/src/java/datafu/pig/sessions/Sessionize.java
deleted file mode 100644
index 52d159b..0000000
--- a/src/java/datafu/pig/sessions/Sessionize.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.sessions;
-
-import java.io.IOException;
-import java.util.UUID;
-
-import org.apache.pig.Accumulator;
-import org.apache.pig.AccumulatorEvalFunc;
-import org.apache.pig.EvalFunc;
-import org.apache.pig.builtin.Nondeterministic;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.joda.time.DateTime;
-import org.joda.time.Period;
-
-/**
- * Sessionizes an input stream, appending a session ID to each tuple.
- *
- * <p>
- * This UDF takes a constructor argument which is the session timeout (an idle
- * period of this amount indicates that a new session has started) and assumes
- * the first element of the input bag is an ISO8601 timestamp. The input bag
- * must be sorted by this timestamp. It returns the input bag with a new field,
- * session_id, that is a GUID indicating the session of the request.
- * </p>
- *
- * <p>
- * Example:
- * <pre>
- * {@code
- *
- * %declare TIME_WINDOW 30m
- *
- * define Sessionize datafu.pig.sessions.Sessionize('$TIME_WINDOW');
- *
- * views = LOAD 'views.tsv' AS (visit_date:chararray, member_id:int, url:chararray);
- *
- * -- sessionize the visit stream
- * views = GROUP views BY member_id;
- * sessions = FOREACH views {
- * visits = ORDER views BY visit_date;
- * GENERATE FLATTEN(Sessionize(VISITS)) AS (visit_date,member_id,url,session_id);
- * }
- *
- * -- count the number of sessions hitting the url
- * rollup = GROUP sessions BY url;
- * result = FOREACH rollup GENERATE group AS url, COUNT(SESSIONS) AS session_cnt;
- * }
- * </pre>
- * </p>
- */
-@Nondeterministic
-public class Sessionize extends AccumulatorEvalFunc<DataBag>
-{
- private final long millis;
-
- private DataBag outputBag;
- private DateTime last_date;
- private String id;
-
- public Sessionize(String timeSpec)
- {
- Period p = new Period("PT" + timeSpec.toUpperCase());
- this.millis = p.toStandardSeconds().getSeconds() * 1000;
-
- cleanup();
- }
-
- @Override
- public void accumulate(Tuple input) throws IOException
- {
- for (Tuple t : (DataBag) input.get(0)) {
- Object timeObj = t.get(0);
-
- DateTime date;
- if (timeObj instanceof String)
- {
- date = new DateTime((String)timeObj);
- }
- else if (timeObj instanceof Long)
- {
- date = new DateTime((Long)timeObj);
- }
- else
- {
- throw new RuntimeException("Time must either be a String or Long");
- }
-
- if (this.last_date == null)
- this.last_date = date;
- else if (date.isAfter(this.last_date.plus(this.millis)))
- this.id = UUID.randomUUID().toString();
- else if (date.isBefore(last_date))
- throw new IOException(String.format("input time series is not sorted (%s < %s)", date, last_date));
-
- Tuple t_new = TupleFactory.getInstance().newTuple(t.getAll());
- t_new.append(this.id);
- outputBag.add(t_new);
-
- this.last_date = date;
- }
- }
-
- @Override
- public DataBag getValue()
- {
- return outputBag;
- }
-
- @Override
- public void cleanup()
- {
- this.last_date = null;
- this.outputBag = BagFactory.getInstance().newDefaultBag();
- this.id = UUID.randomUUID().toString();
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try {
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.BAG)
- {
- throw new RuntimeException("Expected a BAG as input");
- }
-
- Schema inputBagSchema = inputFieldSchema.schema;
-
- if (inputBagSchema.getField(0).type != DataType.TUPLE)
- {
- throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
- DataType.findTypeName(inputBagSchema.getField(0).type)));
- }
-
- Schema inputTupleSchema = inputBagSchema.getField(0).schema;
-
- if (inputTupleSchema.getField(0).type != DataType.CHARARRAY
- && inputTupleSchema.getField(0).type != DataType.LONG)
- {
- throw new RuntimeException(String.format("Expected first element of tuple to be a CHARARRAY or LONG, but instead found %s",
- DataType.findTypeName(inputTupleSchema.getField(0).type)));
- }
-
- Schema outputTupleSchema = inputTupleSchema.clone();
- outputTupleSchema.add(new Schema.FieldSchema("session_id", DataType.CHARARRAY));
-
- return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
- .getName()
- .toLowerCase(), input),
- outputTupleSchema,
- DataType.BAG));
- }
- catch (CloneNotSupportedException e) {
- throw new RuntimeException(e);
- }
- catch (FrontendException e) {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/sessions/package-info.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/sessions/package-info.java b/src/java/datafu/pig/sessions/package-info.java
deleted file mode 100644
index 686fbf0..0000000
--- a/src/java/datafu/pig/sessions/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * UDFs for sessionizing data.
- */
-package datafu.pig.sessions;
\ No newline at end of file
[06/19] DATAFU-27 Migrate build system to Gradle
Posted by wv...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/QuantileUtil.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/QuantileUtil.java b/src/java/datafu/pig/stats/QuantileUtil.java
deleted file mode 100644
index c6fd36a..0000000
--- a/src/java/datafu/pig/stats/QuantileUtil.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats;
-
-import java.util.ArrayList;
-
-/**
- * Methods used by {@link Quantile}.
- *
- * @author "Matthew Hayes <mh...@linkedin.com>"
- *
- */
-public class QuantileUtil
-{
- public static ArrayList<Double> getNQuantiles(int numQuantiles)
- {
- ArrayList<Double> quantiles = new ArrayList<Double>(numQuantiles);
- quantiles = new ArrayList<Double>(numQuantiles);
- int divisor = numQuantiles-1;
- for (int q = 0; q <= divisor; q++)
- {
- double quantile = ((double)q)/divisor;
- quantiles.add(quantile);
- }
- return quantiles;
- }
-
- public static ArrayList<Double> getQuantilesFromParams(String... k)
- {
- ArrayList<Double> quantiles = new ArrayList<Double>(k.length);
- for (String s : k) {
- quantiles.add(Double.parseDouble(s));
- }
-
- if (quantiles.size() == 1 && quantiles.get(0) > 1.0)
- {
- int numQuantiles = Integer.parseInt(k[0]);
- if (numQuantiles < 1)
- {
- throw new IllegalArgumentException("Number of quantiles must be greater than 1");
- }
-
- quantiles = getNQuantiles(numQuantiles);
- }
- else
- {
- for (Double d : quantiles)
- {
- if (d < 0.0 || d > 1.0)
- {
- throw new IllegalArgumentException("Quantile must be between 0.0 and 1.0");
- }
- }
- }
-
- return quantiles;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/StreamingMedian.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/StreamingMedian.java b/src/java/datafu/pig/stats/StreamingMedian.java
deleted file mode 100644
index c4c3be4..0000000
--- a/src/java/datafu/pig/stats/StreamingMedian.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats;
-
-/**
- * Computes the approximate {@link <a href="http://en.wikipedia.org/wiki/Median" target="_blank">median</a>}
- * for a (not necessarily sorted) input bag, using the Munro-Paterson algorithm.
- * This is a convenience wrapper around StreamingQuantile.
- *
- * <p>
- * N.B., all the data is pushed to a single reducer per key, so make sure some partitioning is
- * done (e.g., group by 'day') if the data is too large. That is, this isn't distributed median.
- * </p>
- *
- * @see StreamingQuantile
- */
-public class StreamingMedian extends StreamingQuantile
-{
- public StreamingMedian()
- {
- super("0.5");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/StreamingQuantile.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/StreamingQuantile.java b/src/java/datafu/pig/stats/StreamingQuantile.java
deleted file mode 100644
index e4a65b4..0000000
--- a/src/java/datafu/pig/stats/StreamingQuantile.java
+++ /dev/null
@@ -1,418 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-
-import org.apache.pig.AccumulatorEvalFunc;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
-
-/**
- * Computes approximate {@link <a href="http://en.wikipedia.org/wiki/Quantile" target="_blank">quantiles</a>}
- * for a (not necessarily sorted) input bag, using the Munro-Paterson algorithm.
- *
- * <p>
- * The algorithm is described here:
- * {@link <a href="http://www.cs.ucsb.edu/~suri/cs290/MunroPat.pdf" target="_blank">http://www.cs.ucsb.edu/~suri/cs290/MunroPat.pdf</a>}
- * </p>
- *
- * <p>
- * The implementation is based on the one in Sawzall, available here:
- * {@link <a href="http://szl.googlecode.com/svn-history/r41/trunk/src/emitters/szlquantile.cc">szlquantile.cc</a>}
- * </p>
- *
- * <p>
- * N.B., all the data is pushed to a single reducer per key, so make sure some partitioning is
- * done (e.g., group by 'day') if the data is too large. That is, this isn't distributed quantiles.
- * </p>
- *
- * <p>
- * Note that unlike datafu's standard Quantile algorithm, the Munro-Paterson algorithm gives
- * <b>approximate</b> quantiles and does not require the input bag to be sorted. Because it implements
- * accumulate, StreamingQuantile can be much more efficient than Quantile for large amounts of data which
- * do not fit in memory. Quantile must spill to disk when the input data is too large to fit in memory,
- * which will contribute to longer runtimes.
- * </p>
- *
- * <p>The constructor takes a single integer argument that specifies the number of evenly-spaced
- * quantiles to compute, e.g.,</p>
- *
- * <ul>
- * <li>StreamingQuantile('3') yields the min, the median, and the max
- * <li>StreamingQuantile('5') yields the min, the 25th, 50th, 75th percentiles, and the max
- * <li>StreamingQuantile('101') yields the min, the max, and all 99 percentiles.
- * </ul>
- *
- * <p>Alternatively the constructor can take the explicit list of quantiles to compute, e.g.</p>
- *
- * <ul>
- * <li>StreamingQuantile('0.0','0.5','1.0') yields the min, the median, and the max
- * <li>StreamingQuantile('0.0','0.25','0.5','0.75','1.0') yields the min, the 25th, 50th, 75th percentiles, and the max
- * </ul>
- *
- * <p>The list of quantiles need not span the entire range from 0.0 to 1.0, nor do they need to be evenly spaced, e.g.</p>
- *
- * <ul>
- * <li>StreamingQuantile('0.5','0.90','0.95','0.99') yields the median, the 90th, 95th, and the 99th percentiles
- * <li>StreamingQuantile('0.0013','0.0228','0.1587','0.5','0.8413','0.9772','0.9987') yields the 0.13th, 2.28th, 15.87th, 50th, 84.13th, 97.72nd, and 99.87th percentiles
- * </ul>
- *
- * <p>Be aware when specifying the list of quantiles in this way that more quantiles may be computed internally than are actually returned.
- * The GCD of the quantiles is found and this determines the number of evenly spaced quantiles to compute. The requested quantiles
- * are then returned from this set. For instance:</p>
- *
- * <ul>
- * <li>If the quantiles 0.2 and 0.6 are requested then the quantiles 0.0, 0.2, 0.4, 0.6, 0.8, and 1.0 are computed
- * because 0.2 is the GCD of 0.2, 0.6, and 1.0.</li>
- * <li>If 0.2 and 0.7 are requested then the quantiles 0.0, 0.1, 0.2, ... , 0.9, 1.0 are computed because 0.1 is the
- * GCD of 0.2, 0.7, and 1.0.</li>
- * <li>If 0.999 is requested the quantiles 0.0, 0.001, 0.002, ... , 0.998, 0.999, 1.0 are computed because 0.001 is
- * the GCD of 0.999 and 1.0.</li>
- * </p>
- * </ul>
- *
- * <p>The error on the approximation goes down as the number of buckets computed goes up.</p>
- *
- * <p>
- * Example:
- * <pre>
- * {@code
- *
- * define Quantile datafu.pig.stats.StreamingQuantile('5');
-
- * -- input: 9,10,2,3,5,8,1,4,6,7
- * input = LOAD 'input' AS (val:int);
- *
- * grouped = GROUP input ALL;
- *
- * -- produces: (1.0,3.0,5.0,8.0,10.0)
- * quantiles = FOREACH grouped generate Quantile(input);
- * }
- * </pre></p>
- *
- * @see StreamingMedian
- * @see Quantile
- */
-public class StreamingQuantile extends AccumulatorEvalFunc<Tuple>
-{
- private final int numQuantiles;
- private final QuantileEstimator estimator;
- private List<Double> quantiles;
-
- // For the output schema, label the quantiles 0, 1, 2, ... n
- // Otherwise label the quantiles based on the quantile value.
- // e.g. 50% quantile 0.5 will be labeled as 0_5
- private boolean ordinalOutputSchema;
-
- public StreamingQuantile(String... k)
- {
- if (k.length == 1 && Double.parseDouble(k[0]) > 1.0)
- {
- this.ordinalOutputSchema = true;
- this.numQuantiles = Integer.parseInt(k[0]);
- }
- else
- {
- this.quantiles = QuantileUtil.getQuantilesFromParams(k);
- this.numQuantiles = getNumQuantiles(this.quantiles);
- }
- this.estimator = new QuantileEstimator(this.numQuantiles);
- }
-
- private static int getNumQuantiles(List<Double> quantiles)
- {
- quantiles = new ArrayList<Double>(quantiles);
- Collections.sort(quantiles);
- int start = 0;
- int end = quantiles.size()-1;
- while (quantiles.get(start) == 0.0) start++;
- while (quantiles.get(end) == 1.0) end--;
- double gcd = 1.0;
- for (int i=end; i>=start; i--)
- {
- gcd = gcd(gcd,quantiles.get(i));
- }
- int numQuantiles = (int)(1/gcd) + 1;
- return numQuantiles;
- }
-
- private static double gcd(double a, double b)
- {
- if (round(a) == 0.0)
- {
- throw new IllegalArgumentException("Quantiles are smaller than the allowed precision");
- }
- if (round(b) == 0.0)
- {
- throw new IllegalArgumentException("Quantiles are smaller than the allowed precision");
- }
- while (round(b) != 0.0)
- {
- double t = b;
- b = a % b;
- a = t;
- }
- return round(a);
- }
-
- private static double round(double d)
- {
- return Math.round(d*100000.0)/100000.0;
- }
-
- @Override
- public void accumulate(Tuple b) throws IOException
- {
- DataBag bag = (DataBag) b.get(0);
- if (bag == null || bag.size() == 0)
- return;
-
- for (Tuple t : bag) {
- Object o = t.get(0);
- if (!(o instanceof Number)) {
- throw new IllegalStateException("bag must have numerical values (and be non-null)");
- }
- estimator.add(((Number) o).doubleValue());
- }
- }
-
- @Override
- public void cleanup()
- {
- estimator.clear();
- }
-
- @Override
- public Tuple getValue()
- {
- Tuple t = TupleFactory.getInstance().newTuple(this.quantiles != null ? this.quantiles.size() : this.numQuantiles);
- try {
- if (this.quantiles == null)
- {
- int j = 0;
- for (double quantileValue : estimator.getQuantiles())
- {
- t.set(j, quantileValue);
- j++;
- }
- }
- else
- {
- HashMap<Double,Double> quantileValues = new HashMap<Double,Double>(this.quantiles.size());
- double quantileKey = 0.0;
- for (double quantileValue : estimator.getQuantiles()) {
- quantileValues.put(round(quantileKey), quantileValue);
- quantileKey += 1.0/(this.numQuantiles-1);
- }
- int j = 0;
- for (double d : this.quantiles)
- {
- Double quantileValue = quantileValues.get(round(d));
- t.set(j, quantileValue);
- j++;
- }
- }
- } catch (IOException e) {
- return null;
- }
- return t;
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- Schema tupleSchema = new Schema();
- if (ordinalOutputSchema)
- {
- for (int i = 0; i < this.numQuantiles; i++)
- {
- tupleSchema.add(new Schema.FieldSchema("quantile_" + i, DataType.DOUBLE));
- }
- }
- else
- {
- for (Double x : this.quantiles)
- tupleSchema.add(new Schema.FieldSchema("quantile_" + x.toString().replace(".", "_"), DataType.DOUBLE));
- }
-
- try {
- return new Schema(new FieldSchema(null, tupleSchema, DataType.TUPLE));
- } catch(FrontendException e) {
- throw new RuntimeException(e);
- }
- }
-
- static class QuantileEstimator
- {
- private static final long MAX_TOT_ELEMS = 1024L * 1024L * 1024L * 1024L;
-
- private final List<List<Double>> buffer = new ArrayList<List<Double>>();
- private final int numQuantiles;
- private final int maxElementsPerBuffer;
- private int totalElements;
- private double min;
- private double max;
-
- public QuantileEstimator(int numQuantiles)
- {
- this.numQuantiles = numQuantiles;
- this.maxElementsPerBuffer = computeMaxElementsPerBuffer();
- }
-
- private int computeMaxElementsPerBuffer()
- {
- double epsilon = 1.0 / (numQuantiles - 1.0);
- int b = 2;
- while ((b - 2) * (0x1L << (b - 2)) + 0.5 <= epsilon * MAX_TOT_ELEMS) {
- ++b;
- }
- return (int) (MAX_TOT_ELEMS / (0x1L << (b - 1)));
- }
-
- private void ensureBuffer(int level)
- {
- while (buffer.size() < level + 1) {
- buffer.add(null);
- }
- if (buffer.get(level) == null) {
- buffer.set(level, new ArrayList<Double>());
- }
- }
-
- private void collapse(List<Double> a, List<Double> b, List<Double> out)
- {
- int indexA = 0, indexB = 0, count = 0;
- Double smaller = null;
- while (indexA < maxElementsPerBuffer || indexB < maxElementsPerBuffer) {
- if (indexA >= maxElementsPerBuffer ||
- (indexB < maxElementsPerBuffer && a.get(indexA) >= b.get(indexB))) {
- smaller = b.get(indexB++);
- } else {
- smaller = a.get(indexA++);
- }
-
- if (count++ % 2 == 0) {
- out.add(smaller);
- }
- }
- a.clear();
- b.clear();
- }
-
- private void recursiveCollapse(List<Double> buf, int level)
- {
- ensureBuffer(level + 1);
-
- List<Double> merged;
- if (buffer.get(level + 1).isEmpty()) {
- merged = buffer.get(level + 1);
- } else {
- merged = new ArrayList<Double>(maxElementsPerBuffer);
- }
-
- collapse(buffer.get(level), buf, merged);
- if (buffer.get(level + 1) != merged) {
- recursiveCollapse(merged, level + 1);
- }
- }
-
- public void add(double elem)
- {
- if (totalElements == 0 || elem < min) {
- min = elem;
- }
- if (totalElements == 0 || max < elem) {
- max = elem;
- }
-
- if (totalElements > 0 && totalElements % (2 * maxElementsPerBuffer) == 0) {
- Collections.sort(buffer.get(0));
- Collections.sort(buffer.get(1));
- recursiveCollapse(buffer.get(0), 1);
- }
-
- ensureBuffer(0);
- ensureBuffer(1);
- int index = buffer.get(0).size() < maxElementsPerBuffer ? 0 : 1;
- buffer.get(index).add(elem);
- totalElements++;
- }
-
- public void clear()
- {
- buffer.clear();
- totalElements = 0;
- }
-
- public List<Double> getQuantiles()
- {
- List<Double> quantiles = new ArrayList<Double>();
- quantiles.add(min);
-
- if (buffer.get(0) != null) {
- Collections.sort(buffer.get(0));
- }
- if (buffer.get(1) != null) {
- Collections.sort(buffer.get(1));
- }
-
- int[] index = new int[buffer.size()];
- long S = 0;
- for (int i = 1; i <= numQuantiles - 2; i++) {
- long targetS = (long) Math.ceil(i * (totalElements / (numQuantiles - 1.0)));
-
- while (true) {
- double smallest = max;
- int minBufferId = -1;
- for (int j = 0; j < buffer.size(); j++) {
- if (buffer.get(j) != null && index[j] < buffer.get(j).size()) {
- if (!(smallest < buffer.get(j).get(index[j]))) {
- smallest = buffer.get(j).get(index[j]);
- minBufferId = j;
- }
- }
- }
-
- long incrementS = minBufferId <= 1 ? 1L : (0x1L << (minBufferId - 1));
- if (S + incrementS >= targetS) {
- quantiles.add(smallest);
- break;
- } else {
- index[minBufferId]++;
- S += incrementS;
- }
- }
- }
-
- quantiles.add(max);
- return quantiles;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/VAR.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/VAR.java b/src/java/datafu/pig/stats/VAR.java
deleted file mode 100644
index 6f22f25..0000000
--- a/src/java/datafu/pig/stats/VAR.java
+++ /dev/null
@@ -1,404 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.pig.Accumulator;
-import org.apache.pig.Algebraic;
-import org.apache.pig.EvalFunc;
-import org.apache.pig.FuncSpec;
-import org.apache.pig.PigException;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataByteArray;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.backend.executionengine.ExecException;
-
-
-/**
-* Generates the {@link <a href="http://en.wikipedia.org/wiki/Variance" target="_blank">Variance</a>}
-* of a set of Values. This UDF uses the fact that variance(x) = average(x^2) - average(x)^2
-* This class implements * {@link org.apache.pig.Algebraic}, so if possible the execution will performed in a distributed fashion.
-* VAR implements the {@link org.apache.pig.Accumulator} interface as well.
-*
-* Input: Bag of int, long, double, float or bytearray
-* Output: Double
-*
-* <p>
-* Example:
-* <pre>
-* define VAR datafu.pig.stats.VAR();
-*
-* -- input: 1,2,3,4,10,5,6,7,8,9
-* input = LOAD 'input' AS (val:int);
-* grouped = GROUP input ALL;
-* variance = FOREACH grouped GENERATE VAR(input.val) AS variance;
-* </pre>
-* </p>
-*/
-public class VAR extends EvalFunc<Double> implements Algebraic, Accumulator<Double> {
- private static TupleFactory mTupleFactory = TupleFactory.getInstance();
-
- @Override
- public Double exec(Tuple input) throws IOException {
- try {
- Double sum = sum(input);
- Double sumSquare = sumSquare(input);
-
- if(sum == null) {
- // either we were handed an empty bag or a bag
- // filled with nulls - return null in this case
- return null;
- }
- long count = count(input);
-
- Double var = null;
- if (count > 0){
- Double avg = new Double(sum / count);
- Double avgSquare = new Double(sumSquare / count);
- var = avgSquare - avg*avg;
- }
-
- return var;
- } catch (ExecException ee) {
- throw ee;
- }
- }
-
- public String getInitial() {
- return Initial.class.getName();
- }
-
- public String getIntermed() {
- return Intermediate.class.getName();
- }
-
- public String getFinal() {
- return Final.class.getName();
- }
-
- static public class Initial extends EvalFunc<Tuple> {
- @Override
- public Tuple exec(Tuple input) throws IOException {
- Tuple t = mTupleFactory.newTuple(3);
- try {
- // input is a bag with one tuple containing
- // the column we are trying to get variance
- DataBag bg = (DataBag) input.get(0);
- DataByteArray dba = null;
- Iterator<Tuple> iter = bg.iterator();
- if(iter.hasNext()) {
- Tuple tp = iter.next();
- dba = (DataByteArray)tp.get(0);
- }
-
- if (iter.hasNext())
- {
- throw new RuntimeException("Expected only one tuple in bag");
- }
-
- Double d = dba !=null ? Double.valueOf(dba.toString()) : null;
- if (dba == null){
- t.set(0, null);
- t.set(1, null);
- t.set(2, 0L);
- }
- else {
- t.set(0, d);
- t.set(1, d*d);
- t.set(2, 1L);
- }
- return t;
- } catch(NumberFormatException nfe) {
- nfe.printStackTrace();
- // invalid input,
- // treat this input as null
- try {
- t.set(0, null);
- t.set(1, null);
- t.set(2, 0L);
- } catch (ExecException e) {
- throw e;
- }
- return t;
- } catch (ExecException ee) {
- ee.printStackTrace();
- throw ee;
- } catch (Exception e) {
- e.printStackTrace();
- int errCode = 2106;
- String msg = "Error while computing variance in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
- }
-
- }
- }
-
- static public class Intermediate extends EvalFunc<Tuple> {
- @Override
- public Tuple exec(Tuple input) throws IOException {
- try {
- DataBag b = (DataBag)input.get(0);
- return combine(b);
- } catch (ExecException ee) {
- throw ee;
- } catch (Exception e) {
- int errCode = 2106;
- String msg = "Error while computing variacne in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
-
- }
- }
- }
-
- static public class Final extends EvalFunc<Double> {
- @Override
- public Double exec(Tuple input) throws IOException {
- try {
- DataBag b = (DataBag)input.get(0);
- Tuple combined = combine(b);
-
- Double sum = (Double)combined.get(0);
- Double sumSquare = (Double)combined.get(1);
- if(sum == null) {
- return null;
- }
- Long count = (Long)combined.get(2);
-
- Double var = null;
-
- if (count > 0) {
- Double avg = new Double(sum / count);
- Double avgSquare = new Double(sumSquare / count);
- var = avgSquare - avg*avg;
- }
- return var;
- } catch (ExecException ee) {
- throw ee;
- } catch (Exception e) {
- int errCode = 2106;
- String msg = "Error while computing variance in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
- }
- }
- }
-
- static protected Tuple combine(DataBag values) throws ExecException{
- double sum = 0;
- double sumSquare = 0;
- long totalCount = 0;
-
- // combine is called from Intermediate and Final
- // In either case, Initial would have been called
- // before and would have sent in valid tuples
- // Hence we don't need to check if incoming bag
- // is empty
-
- Tuple output = mTupleFactory.newTuple(3);
- boolean sawNonNull = false;
- for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
- Tuple t = it.next();
- Double d = (Double)t.get(0);
- Double dSquare = (Double)t.get(1);
- Long count = (Long)t.get(2);
-
- // we count nulls in var as contributing 0
- // a departure from SQL for performance of
- // COUNT() which implemented by just inspecting
- // size of the bag
- if(d == null) {
- d = 0.0;
- } else {
- sawNonNull = true;
- }
- sum += d;
- sumSquare += dSquare;
- totalCount += count;
- }
- if(sawNonNull) {
- output.set(0, new Double(sum));
- output.set(1, new Double(sumSquare));
- } else {
- output.set(0, null);
- output.set(1, null);
- }
- output.set(2, Long.valueOf(totalCount));
- return output;
- }
-
- static protected long count(Tuple input) throws ExecException {
- DataBag values = (DataBag)input.get(0);
- long cnt = 0;
- Iterator<Tuple> it = values.iterator();
- while (it.hasNext()){
- Tuple t = (Tuple)it.next();
- if (t != null && t.size() > 0 && t.get(0) != null)
- cnt ++;
- }
-
- return cnt;
- }
-
- static protected Double sum(Tuple input) throws ExecException, IOException {
- DataBag values = (DataBag)input.get(0);
-
- // if we were handed an empty bag, return NULL
- if(values.size() == 0) {
- return null;
- }
-
- double sum = 0;
- boolean sawNonNull = false;
- for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
- Tuple t = it.next();
- try{
- DataByteArray dba = (DataByteArray)t.get(0);
- Double d = dba != null ? Double.valueOf(dba.toString()) : null;
- if (d == null) continue;
- sawNonNull = true;
- sum += d;
- }catch(RuntimeException exp) {
- int errCode = 2103;
- String msg = "Problem while computing sum of values.";
- throw new ExecException(msg, errCode, PigException.BUG, exp);
- }
- }
-
- if(sawNonNull) {
- return new Double(sum);
- } else {
- return null;
- }
- }
-
- static protected Double sumSquare(Tuple input) throws ExecException, IOException {
- DataBag values = (DataBag)input.get(0);
-
- // if we were handed an empty bag, return NULL
- if(values.size() == 0) {
- return null;
- }
-
- double sumSquare = 0;
- boolean sawNonNull = false;
- for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
- Tuple t = it.next();
- try{
- DataByteArray dba = (DataByteArray)t.get(0);
- Double d = dba != null ? Double.valueOf(dba.toString()) : null;
- if (d == null) continue;
- sawNonNull = true;
- sumSquare += d*d;
- }catch(RuntimeException exp) {
- int errCode = 2103;
- String msg = "Problem while computing sum of squared values.";
- throw new ExecException(msg, errCode, PigException.BUG, exp);
- }
- }
-
- if(sawNonNull) {
- return new Double(sumSquare);
- } else {
- return null;
- }
- }
-
- @Override
- public Schema outputSchema(Schema input) {
- return new Schema(new Schema.FieldSchema(null, DataType.DOUBLE));
- }
-
- @Override
- public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
- List<FuncSpec> funcList = new ArrayList<FuncSpec>();
- funcList.add(new FuncSpec(this.getClass().getName(), Schema.generateNestedSchema(DataType.BAG, DataType.BYTEARRAY)));
- funcList.add(new FuncSpec(DoubleVAR.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.DOUBLE)));
- funcList.add(new FuncSpec(FloatVAR.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.FLOAT)));
- funcList.add(new FuncSpec(IntVAR.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.INTEGER)));
- funcList.add(new FuncSpec(LongVAR.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.LONG)));
- return funcList;
- }
-
- /* Accumulator interface implementation */
- private Double intermediateSumSquare = null;
- private Double intermediateSum = null;
- private Long intermediateCount = null;
-
- @Override
- public void accumulate(Tuple b) throws IOException {
- try {
- Double sum = sum(b);
- if(sum == null) {
- return;
- }
-
- Double sumSquare = sumSquare(b);
- if(sumSquare == null) {
- return;
- }
-
- // set default values
- if (intermediateSum == null || intermediateCount == null) {
- intermediateSumSquare = 0.0;
- intermediateSum = 0.0;
- intermediateCount = (long) 0;
- }
-
- long count = (Long)count(b);
-
- if (count > 0) {
- intermediateCount += count;
- intermediateSum += sum;
- intermediateSumSquare += sumSquare;
- }
- } catch (ExecException ee) {
- throw ee;
- } catch (Exception e) {
- int errCode = 2106;
- String msg = "Error while computing variance in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
- }
- }
-
- @Override
- public void cleanup() {
- intermediateSumSquare = null;
- intermediateSum = null;
- intermediateCount = null;
- }
-
- @Override
- public Double getValue() {
- Double var = null;
- if (intermediateCount != null && intermediateCount > 0) {
- Double avg = new Double(intermediateSum / intermediateCount);
- Double avgSquare = new Double(intermediateSumSquare / intermediateCount);
- var = avgSquare - avg*avg;
- }
- return var;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/WilsonBinConf.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/WilsonBinConf.java b/src/java/datafu/pig/stats/WilsonBinConf.java
deleted file mode 100644
index 1448611..0000000
--- a/src/java/datafu/pig/stats/WilsonBinConf.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.commons.math.MathException;
-import org.apache.commons.math.distribution.NormalDistribution;
-import org.apache.commons.math.distribution.NormalDistributionImpl;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
-
-import datafu.pig.util.SimpleEvalFunc;
-
-/**
- * Computes the {@link <a href="http://en.wikipedia.org/wiki/Binomial_proportion_confidence_interval#Wilson_score_interval" target="_blank">Wilsonian binomial proportion confidence interval</a>}
- * <p>
- * Constructor requires the confidence interval (alpha) parameter, and the
- * parameters are the number of positive (success) outcomes and the total
- * number of observations. The UDF returns the (lower,upper) confidence
- * interval.
- * <p>
- * Example:
- * <pre>
- * {@code
- * -- the Wilsonian binomial proportion confidence interval for scoring
- * %declare WILSON_ALPHA 0.10
- *
- * define WilsonBinConf datafu.pig.stats.WilsonBinConf('$WILSON_ALPHA');
- *
- * bar = FOREACH foo GENERATE WilsonBinConf(successes, totals).lower as score;
- * quux = ORDER bar BY score DESC;
- * top = LIMIT quux 10;
- * }
- * </pre></p>
- */
-public class WilsonBinConf extends SimpleEvalFunc<Tuple>
-{
- private static TupleFactory tupleFactory = TupleFactory.getInstance();
- private final double alpha;
-
- public WilsonBinConf(double alpha)
- {
- this.alpha = alpha;
- }
-
- public WilsonBinConf(String alpha)
- {
- this(Double.parseDouble(alpha));
- }
-
- public Tuple call(Number x, Number n) throws IOException
- {
- if (x == null || n == null)
- return null;
- return binconf(x.longValue(), n.longValue());
- }
-
- /**
- * @param x The number of positive (success) outcomes
- * @param n The number of observations
- * @return The (lower,upper) confidence interval
- */
- public Tuple binconf(Long x, Long n) throws IOException
- {
- NormalDistribution normalDist = new NormalDistributionImpl();
-
- if (x == null || n == null)
- return null;
- if (x < 0 || n < 0)
- throw new IllegalArgumentException("non-negative values expected");
- if (x > n)
- throw new IllegalArgumentException("invariant violation: number of successes > number of obs");
- if (n == 0)
- return tupleFactory.newTuple(Arrays.asList(Double.valueOf(0), Double.valueOf(0)));
-
- try {
- double zcrit = -1.0 * normalDist.inverseCumulativeProbability(alpha/2);
- double z2 = zcrit * zcrit;
- double p = x/(double)n;
-
- double a = p + z2/2/n;
- double b = zcrit * Math.sqrt((p * (1 - p) + z2/4/n)/n);
- double c = (1 + z2/n);
-
- double lower = (a - b) / c;
- double upper = (a + b) / c;
-
- // Add corrections for when x is very close to n. This improves the estimates.
- // For more info on wilson binomial confidence interval, see paper:
- // L.D. Brown, T.T. Cai and A. DasGupta, Interval estimation for a binomial proportion (with discussion),
- // _Statistical Science,_*16*:101-133, 2001.
- // http://www-stat.wharton.upenn.edu/~tcai/paper/Binomial-StatSci.pdf
-
- if (x == 1)
- lower = -Math.log(1 - alpha)/n;
- if (x == (n - 1))
- upper = 1 + Math.log(1 - alpha)/n;
-
- return tupleFactory.newTuple(Arrays.asList(lower, upper));
- }
- catch (MathException e) {
- throw new IOException("math error", e);
- }
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try {
- Schema innerSchema = new Schema(Arrays.asList(
- new Schema.FieldSchema("lower", DataType.DOUBLE),
- new Schema.FieldSchema("upper", DataType.DOUBLE)));
-
- return new Schema(new FieldSchema(null, innerSchema, DataType.TUPLE));
- } catch(FrontendException e) {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/entropy/ChaoShenEntropyEstimator.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/entropy/ChaoShenEntropyEstimator.java b/src/java/datafu/pig/stats/entropy/ChaoShenEntropyEstimator.java
deleted file mode 100644
index bfb7398..0000000
--- a/src/java/datafu/pig/stats/entropy/ChaoShenEntropyEstimator.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats.entropy;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.backend.executionengine.ExecException;
-
-import java.util.Collections;
-
-
-
-/*
- * This class implements the Chao-Shen entropy estimator based on this
- * {@link <a href="http://www.researchgate.net/publication/226041346_Nonparametric_estimation_of_Shannons_index_of_diversity_when_there_are_unseen_species_in_sample/file/d912f510a79b8aae74.pdf" target="_blank">paper</a>}
- * <p>
- * It combines the {@link <a href="http://en.wikipedia.org/wiki/Horvitz%E2%80%93Thompson_estimator" target="_blank">Horvitz-Thompson estimator</a>}
- * to adjust for missing species and the concept of sample coverage to estimate the relative abundances of species in the sample.
- * The entropy calculation formula is as follows:
- * <pre>
- * {@code
- * H(X) = - SUM( ( C * c(x) / N * log(C * c(x) / N) ) / ( 1 - (1 - C * c(x) / N) ^ N ) )
- * c(x) is the occurrence frequency of sample x, N is the sum of c(x)
- * C = 1 - N1 / N, N1 is the number of samples whose c(x) equals 1.
- * }
- * </pre>
- * </p>
- */
-class ChaoShenEntropyEstimator extends EntropyEstimator {
- //sample frequency
- private AccumulativeSampleFrequencyMap freqBaggedMap;
-
- //sum of frequency of all samples
- private long N;
-
- //number of samples whose occurrence frequency is 1
- private long N1;
-
- ChaoShenEntropyEstimator(String base){
- super(base);
- reset();
- }
-
- @Override
- public void accumulate(long cx) throws ExecException {
- if(cx > 0) {
- freqBaggedMap.accumulate(cx);
- N += cx;
- if(cx == 1) {
- N1++;
- }
- }
- }
-
- @Override
- public double getEntropy() {
- double h = 0.0;
-
- if(N > 0) {
-
- if(N1 == N) {
- //avoid c == 0
- N1 = N - 1;
- }
-
- //sample coverage estimation
- double c = 1 - (double)N1 / N;
-
- try {
- //from in-memory frequency map
- for(Iterator<Map.Entry<Long, MutableLong>> iter = this.freqBaggedMap.getInternalMap().entrySet().iterator();
- iter.hasNext(); ) {
- Map.Entry<Long, MutableLong> entry = iter.next();
- long cx = entry.getKey();
- long cnt = entry.getValue().longValue();
- h += accumlateEntropy(cx, this.N, c, cnt);
- }
- //from backup databag
- for(Iterator<Tuple> iter = this.freqBaggedMap.getInternalBag().iterator();
- iter.hasNext(); ) {
- Tuple t = iter.next();
- long cx = (Long)t.get(0);
- long cnt = (Long)t.get(1);
- h += accumlateEntropy(cx, this.N, c, cnt);
- }
- } catch(ExecException ex) {
- throw new RuntimeException(
- "Error while computing chao-shen entropy, exception: " + ex);
- }
- }
-
- return EntropyUtil.logTransform(h, super.base);
- }
-
- private double accumlateEntropy(long cx,
- long N,
- double c,
- long cnt) {
- //sample proportion
- double p = (double)cx / N;
- //sample coverage adjusted probability
- double pa = c * p;
- //probability to see an individual in the sample
- double la = 1 - Math.pow((1 - pa), N);
- return -(cnt * pa * Math.log(pa) / la);
- }
-
- @Override
- public void reset() {
- this.N = 0;
- this.N1 = 0;
- if(this.freqBaggedMap != null) {
- this.freqBaggedMap.clear();
- }
- this.freqBaggedMap = new AccumulativeSampleFrequencyMap();
- }
-
- /*
- * A map backed by a data bag to record the sample occurrence frequencies which might be repeated
- * The purpose of this class is that we suspect in the real applications, the sample occurrence frequency
- * might be repeated and putting all these repeated numbers into databag may be space inefficient
- * So we have an in-memory map to record the repetitions and use a databag to backup when the size of the
- * in-memory map exceeds a pre-defined threshold. At present we use 5M as the default threshold to control
- * the number of entries in the in-memory map, which approximates to 80M bytes.
- */
- private class AccumulativeSampleFrequencyMap {
-
- private long spillBytesThreshold;
-
- /* key is the sample occurrence frequency
- * value is the number of repetitions of this frequency
- */
- private Map<Long, MutableLong> countMap;
-
- /* the backed databag
- * each tuple has 2 elements
- * the 1st element is sample occurrence frequency
- * the 2nd element is the number of repetitions
- */
- private DataBag countBag;
-
- AccumulativeSampleFrequencyMap(){
- this(1024 * 1024 * 5 * (Long.SIZE + Long.SIZE) / 8);
- }
-
- AccumulativeSampleFrequencyMap(long spillBytesThreshold) {
- this.spillBytesThreshold = spillBytesThreshold;
- clear();
- }
-
- void accumulate(long key) throws ExecException {
- MutableLong val = this.countMap.get(key);
- if(val == null) {
- this.countMap.put(key, new MutableLong(1));
- } else {
- val.add(1);
- }
-
- if(this.countMap.size() * (Long.SIZE + Long.SIZE) / 8 > spillBytesThreshold) {
- spillFromMap2Bag();
- }
- }
-
- private void spillFromMap2Bag() throws ExecException {
- for(Map.Entry<Long, MutableLong> entry : this.countMap.entrySet()) {
- Tuple t = TupleFactory.getInstance().newTuple(2);
- t.set(0, entry.getKey());
- t.set(1, entry.getValue().longValue());
- this.countBag.add(t);
- }
- this.countMap.clear();
- }
-
- Map<Long, MutableLong> getInternalMap() {
- return Collections.unmodifiableMap(this.countMap);
- }
-
- DataBag getInternalBag() {
- return this.countBag;
- }
-
- void clear() {
- countMap = new HashMap<Long, MutableLong>();
- countBag = BagFactory.getInstance().newDefaultBag();
- }
- }
-
- private class MutableLong {
- private long val;
-
- MutableLong(){
- this(0L);
- }
-
- MutableLong(long val) {
- this.val = val;
- }
-
- long longValue(){
- return val;
- }
-
- void add(long additive){
- this.val += additive;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/entropy/CondEntropy.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/entropy/CondEntropy.java b/src/java/datafu/pig/stats/entropy/CondEntropy.java
deleted file mode 100644
index 26b743e..0000000
--- a/src/java/datafu/pig/stats/entropy/CondEntropy.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats.entropy;
-
-import org.apache.pig.AccumulatorEvalFunc;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-
-
-/**
- * Calculate conditional entropy H(Y|X) of random variables X and Y following conditional entropy's
- * {@link <a href="http://en.wikipedia.org/wiki/Conditional_entropy" target="_blank">wiki definition</a>},
- * X is the conditional variable and Y is the variable that conditions on X.
- * <p>
- * Each tuple of the input bag has 2 fields, the 1st field is an object instance of variable X and
- * the 2nd field is an object instance of variable Y. An exception will be thrown if the number of fields is not 2.
- * </p>
- * <p>
- * This UDF's constructor definition and parameters are the same as that of {@link datafu.pig.stats.entropy.Entropy}
- * </p>
- * <p>
- * Note:
- * <ul>
- * <li>The input bag to this UDF must be <b>sorted</b> on X and Y, with X in the first sort order.
- * An exception will be thrown if the input bag is not sorted.
- * <li>The returned entropy value is of double type.
- * </ul>
- * </p>
- * <p>
- * How to use:
- * </p>
- * <p>
- * This UDF calculates conditional entropy given raw data tuples of X and Y without the need to pre-compute per tuple occurrence frequency.
- * </p>
- * <p>
- * It could be used in a nested FOREACH after a GROUP BY, in which we sort the inner bag and use the sorted bag as this UDF's input.
- * </p>
- * <p>
- * Example:
- * <pre>
- * {@code
- * --define empirical conditional entropy with Euler's number as the logarithm base
- * define CondEntropy datafu.pig.stats.entropy.CondEntropy();
- *
- * input = LOAD 'input' AS (grp: chararray, valX: double, valY: double);
- *
- * -- calculate conditional entropy H(Y|X) in each group
- * input_group_g = GROUP input BY grp;
- * entropy_group = FOREACH input_group_g {
- * input_val = input.(valX, valY)
- * input_ordered = ORDER input_val BY $0, $1;
- * GENERATE FLATTEN(group) AS group, CondEntropy(input_ordered) AS cond_entropy;
- * }
- * }
- * </pre>
- * </p>
- * Use case to calculate mutual information:
- * <p>
- * <pre>
- * {@code
- * ------------
- * -- calculate mutual information I(X, Y) using conditional entropy UDF and entropy UDF
- * -- I(X, Y) = H(Y) - H(Y|X)
- * ------------
- *
- * define CondEntropy datafu.pig.stats.entropy.CondEntropy();
- * define Entropy datafu.pig.stats.entropy.Entropy();
- *
- * input = LOAD 'input' AS (grp: chararray, valX: double, valY: double);
- *
- * -- calculate the I(X,Y) in each group
- * input_group_g = GROUP input BY grp;
- * mutual_information = FOREACH input_group_g {
- * input_val_x_y = input.(valX, valY);
- * input_val_x_y_ordered = ORDER input_val_x_y BY $0,$1;
- * input_val_y = input.valY;
- * input_val_y_ordered = ORDER input_val_y BY $0;
- * cond_h_x_y = CondEntropy(input_val_x_y_ordered);
- * h_y = Entropy(input_val_y_ordered);
- * GENERATE FLATTEN(group), h_y - cond_h_x_y;
- * }
- * }
- * </pre>
- * </p>
- * @see Entropy
- */
-public class CondEntropy extends AccumulatorEvalFunc<Double> {
- //last visited tuple of <x,y>
- private Tuple xy;
-
- //number of occurrence of last visited <x,y>
- private long cxy;
-
- //number of occurrence of last visited x
- private long cx;
-
- //comparison result between the present tuple and the last visited tuple
- private int lastCmp;
-
- //entropy estimator for H(x,y)
- private EntropyEstimator combEstimator;
-
- //entropy estimator for H(x)
- private EntropyEstimator condXEstimator;
-
- public CondEntropy() throws ExecException
- {
- this(EntropyEstimator.EMPIRICAL_ESTIMATOR);
- }
-
- public CondEntropy(String type) throws ExecException
- {
- this(type, EntropyUtil.LOG);
- }
-
- public CondEntropy(String type, String base) throws ExecException
- {
- try {
- this.combEstimator = EntropyEstimator.createEstimator(type, base);
- this.condXEstimator = EntropyEstimator.createEstimator(type, base);
- } catch (IllegalArgumentException ex) {
- throw new ExecException(String.format(
- "Fail to initialize StreamingCondEntropy with entropy estimator of type (%s), base: (%s). Exception: (%s)",
- type, base, ex));
- }
- cleanup();
- }
-
- /*
- * Accumulate occurrence frequency of <x,y> and x
- * as we stream through the input bag
- */
- @Override
- public void accumulate(Tuple input) throws IOException
- {
- for (Tuple t : (DataBag) input.get(0)) {
-
- if (this.xy != null)
- {
- int cmp = t.compareTo(this.xy);
-
- //check if the comparison result is different from previous compare result
- if ((cmp < 0 && this.lastCmp > 0)
- || (cmp > 0 && this.lastCmp < 0)) {
- throw new ExecException("Out of order! previous tuple: " + this.xy + ", present tuple: " + t
- + ", comparsion: " + cmp + ", previous comparsion: " + this.lastCmp);
- }
- if (cmp != 0) {
- //different <x,y>
- this.combEstimator.accumulate(this.cxy);
- this.cxy = 0;
- this.lastCmp = cmp;
- if(DataType.compare(this.xy.get(0), t.get(0)) != 0) {
- //different x
- this.condXEstimator.accumulate(this.cx);
- this.cx = 0;
- }
- }
- }
-
- //set tuple t as the next tuple for comparison
- this.xy = t;
-
- //accumulate cx
- this.cx++;
-
- //accumulate cxy
- this.cxy++;
- }
- }
-
- @Override
- public Double getValue()
- {
- //do not miss the last tuple
- try {
- this.combEstimator.accumulate(this.cxy);
- this.condXEstimator.accumulate(this.cx);
- } catch (ExecException ex) {
- throw new RuntimeException("Error while accumulating sample frequency: " + ex);
- }
-
- //Chain rule: H(Y|X) = H(X, Y) - H(X)
- return this.combEstimator.getEntropy() - this.condXEstimator.getEntropy();
- }
-
- @Override
- public void cleanup()
- {
- this.xy = null;
- this.cxy = 0;
- this.cx = 0;
- this.lastCmp = 0;
- this.combEstimator.reset();
- this.condXEstimator.reset();
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try {
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.BAG)
- {
- throw new RuntimeException("Expected a BAG as input");
- }
-
- Schema inputBagSchema = inputFieldSchema.schema;
-
- if (inputBagSchema.getField(0).type != DataType.TUPLE)
- {
- throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
- DataType.findTypeName(inputBagSchema.getField(0).type)));
- }
-
- Schema tupleSchema = inputBagSchema.getField(0).schema;
-
- if(tupleSchema == null) {
- throw new RuntimeException("The tuple of the input bag has no schema");
- }
-
- List<Schema.FieldSchema> fieldSchemaList = tupleSchema.getFields();
-
- if(fieldSchemaList == null || fieldSchemaList.size() != 2) {
- throw new RuntimeException("The field schema of the input tuple is null or its size is not 2");
- }
-
- return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
- .getName()
- .toLowerCase(), input),
- DataType.DOUBLE));
- } catch (FrontendException e) {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/entropy/EmpiricalCountEntropy.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/entropy/EmpiricalCountEntropy.java b/src/java/datafu/pig/stats/entropy/EmpiricalCountEntropy.java
deleted file mode 100644
index 388b80f..0000000
--- a/src/java/datafu/pig/stats/entropy/EmpiricalCountEntropy.java
+++ /dev/null
@@ -1,425 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats.entropy;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.pig.AccumulatorEvalFunc;
-import org.apache.pig.Algebraic;
-import org.apache.pig.EvalFunc;
-import org.apache.pig.PigException;
-import org.apache.pig.PigWarning;
-import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-import datafu.pig.stats.entropy.EntropyUtil;
-
-
-/**
- * Calculate the empirical entropy of random variable X given its occurrence frequencies, following entropy's
- * {@link <a href="http://en.wikipedia.org/wiki/Entropy_%28information_theory%29" target="_blank">wiki definition</a>}
- * <p>
- * This UDF's constructor takes 1 argument: the logarithm base, whose definition is the same as that defined in {@link datafu.pig.stats.entropy.Entropy}
- * </p>
- * <p>
- * Note:
- * <ul>
- * <li>Unlike {@link datafu.pig.stats.entropy.Entropy}, which calculates entropy from sorted raw data bag in accumulative mode,
- * this UDF calculates entropy from the data's occurrence frequencies which does not need to be sorted, either in accumulative or algebraic mode.
- * <li>Each tuple of the UDF's input bag <b>must only</b> have 1 field, the occurrence frequency of a data instance,
- * and the data type of this field <b>must</b> be int or long. Otherwise, an exception will be thrown.
- * <li>Negative frequency number will be silently discarded and a warning message will be logged in the job's log file.
- * <li>The returned entropy value is of double type.
- * </ul>
- * </p>
- * <p>
- * How to use:
- * </p>
- * <p>
- * To use this UDF, customer needs to pre-compute the occurrence frequency of each data instance, often in an outer GROUP BY
- * , and then use this UDF to calculate entropy with those frequency numbers in another outer GROUP BY.
- * </p>
- * <p>
- * Compared with {@link datafu.pig.stats.entropy.Entropy}, this UDF is more scalable when we need to handle a very large data set,
- * since it could distribute computation onto mappers and take advantage of combiners to reduce intermedidate output from mappers to reducers.
- * </p>
- * <p>
- * Example:
- * <pre>
- * {@code
- *
- * define Entropy datafu.pig.stats.entropy.EmpiricalCountEntropy();
- *
- * input = LOAD 'input' AS (val: double);
- *
- * -- calculate the occurrence of each instance
- * counts_g = GROUP input BY val;
- * counts = FOREACh counts_g GENERATE COUNT(input) AS cnt;
- *
- * -- calculate entropy
- * input_counts_g = GROUP counts ALL;
- * entropy = FOREACH input_counts_g GENERATE Entropy(counts) AS entropy;
- * }
- * </pre>
- * </p>
- * Use case to calculate mutual information using EmpiricalCountEntropy:
- * <p>
- * <pre>
- * {@code
- *
- * define Entropy datafu.pig.stats.entropy.EmpiricalCountEntropy();
- *
- * input = LOAD 'input' AS (valX: double, valY: double);
- *
- * ------------
- * -- calculate mutual information I(X, Y) using entropy
- * -- I(X, Y) = H(X) + H(Y) - H(X, Y)
- * ------------
- *
- * input_x_y_g = GROUP input BY (valX, valY);
- * input_x_y_cnt = FOREACH input_x_y_g GENERATE flatten(group) as (valX, valY), COUNT(input) AS cnt;
- *
- * input_x_g = GROUP input_x_y_cnt BY valX;
- * input_x_cnt = FOREACH input_x_g GENERATE flatten(group) as valX, SUM(input_x_y_cnt.cnt) AS cnt;
- *
- * input_y_g = GROUP input_x_y_cnt BY valY;
- * input_y_cnt = FOREACH input_y_g GENERATE flatten(group) as valY, SUM(input_x_y_cnt.cnt) AS cnt;
- *
- * input_x_y_entropy_g = GROUP input_x_y_cnt ALL;
- * input_x_y_entropy = FOREACH input_x_y_entropy_g {
- * input_x_y_entropy_cnt = input_x_y_cnt.cnt;
- * GENERATE Entropy(input_x_y_entropy_cnt) AS x_y_entropy;
- * }
- *
- * input_x_entropy_g = GROUP input_x_cnt ALL;
- * input_x_entropy = FOREACH input_x_entropy_g {
- * input_x_entropy_cnt = input_x_cnt.cnt;
- * GENERATE Entropy(input_x_entropy_cnt) AS x_entropy;
- * }
- *
- * input_y_entropy_g = GROUP input_y_cnt ALL;
- * input_y_entropy = FOREACH input_y_entropy_g {
- * input_y_entropy_cnt = input_y_cnt.cnt;
- * GENERATE Entropy(input_y_entropy_cnt) AS y_entropy;
- * }
- *
- * input_mi_cross = CROSS input_x_y_entropy, input_x_entropy, input_y_entropy;
- * input_mi = FOREACH input_mi_cross GENERATE (input_x_entropy::x_entropy +
- * input_y_entropy::y_entropy -
- * input_x_y_entropy::x_y_entropy) AS mi;
- * }
- * </pre>
- * </p>
- * @see Entropy
- */
-
-public class EmpiricalCountEntropy extends AccumulatorEvalFunc<Double> implements Algebraic {
-
- private static TupleFactory mTupleFactory = TupleFactory.getInstance();
-
- //entropy estimator for accumulator
- //re-use the same entropy estimator for StreamingEntropy
- private EntropyEstimator streamEstimator;
-
- //logarithm base
- private String base;
-
- public EmpiricalCountEntropy() throws ExecException {
- //empirical estimator using Euler's number as logarithm base
- this(EntropyUtil.LOG);
- }
-
- public EmpiricalCountEntropy(String base) throws ExecException {
- try {
- this.streamEstimator = EntropyEstimator.createEstimator(EntropyEstimator.EMPIRICAL_ESTIMATOR, base);
- } catch (IllegalArgumentException ex) {
- throw new ExecException(
- String.format("Fail to initialize EmpiricalCountEntropy with logarithm base: (%s), exception: (%s)", base, ex));
- }
- this.base = base;
- }
-
- /*
- * Algebraic implementation part
- */
-
- private String param = null;
- private String getParam()
- {
- if (param == null) {
- if (this.base != null) {
- param = String.format("('%s')", this.base);
- } else {
- param = "";
- }
- }
- return param;
- }
-
- @Override
- public String getFinal() {
- return Final.class.getName() + getParam();
- }
-
- @Override
- public String getInitial() {
- return Initial.class.getName() + getParam();
- }
-
- @Override
- public String getIntermed() {
- return Intermediate.class.getName() + getParam();
- }
-
- static public class Initial extends EvalFunc<Tuple> {
-
- public Initial(){}
-
- public Initial(String base){}
-
- @Override
- public Tuple exec(Tuple input) throws IOException {
- Tuple t = mTupleFactory.newTuple(2);
-
- try{
- //input is a bag with one tuple containing
- //the sample's occurrence frequency
- DataBag bg = (DataBag) input.get(0);
- Long cxl = null;
- if(bg.iterator().hasNext()) {
- Tuple tp = bg.iterator().next();
- cxl = ((Number)(tp.get(0))).longValue();
- }
-
- if(cxl == null || cxl.longValue() < 0) {
- //emit null in case of invalid input frequency
- t.set(0, null);
- t.set(1, null);
- warn("Non-positive input frequency number: " + cxl, PigWarning.UDF_WARNING_1);
- } else {
- long cx = cxl.longValue();
- double logcx = (cx > 0 ? Math.log(cx) : 0);
- double cxlogcx = cx * logcx;
-
- //1st element of the returned tuple is freq * log(freq)
- t.set(0, cxlogcx);
-
- //2nd element of the returned tuple is freq
- t.set(1, cxl);
- }
-
- return t;
- } catch (ExecException ee) {
- throw ee;
- } catch(Exception e) {
- int errCode = 10080;
- String msg = "Error while computing entropy in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
- }
-
- }
-
- }
-
- static public class Intermediate extends EvalFunc<Tuple> {
-
- public Intermediate(){}
-
- public Intermediate(String base){}
-
- @Override
- public Tuple exec(Tuple input) throws IOException {
- try {
- DataBag b = (DataBag)input.get(0);
- return combine(b);
- } catch (ExecException ee) {
- throw ee;
- } catch (Exception e) {
- int errCode = 10081;
- String msg = "Error while computing entropy in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
- }
- }
- }
-
- static public class Final extends EvalFunc<Double> {
- private String base;
-
- public Final()
- {
- this(EntropyUtil.LOG);
- }
-
- public Final(String base)
- {
- this.base = base;
- }
-
- @Override
- public Double exec(Tuple input) throws IOException {
- try {
- DataBag b = (DataBag)input.get(0);
- Tuple combined = combine(b);
-
- Double sumOfCxLogCx = (Double)combined.get(0);
- Long sumOfCx = (Long)combined.get(1);
-
- if(sumOfCxLogCx == null || sumOfCx == null) {
- //emit null if there is at least 1 invalid input
- warn("Invalid null field output from combine(), " +
- "1st field: " + sumOfCxLogCx + ", 2nd field: " + sumOfCx, PigWarning.UDF_WARNING_1);
- return null;
- }
-
- Double entropy = null;
-
- double scxlogcx = sumOfCxLogCx.doubleValue();
- long scx = sumOfCx.longValue();
-
- if (scx > 0) {
- //H(X) = log(N) - 1 / N * SUM(c(x) * log(c(x)) )
- entropy = EntropyUtil.logTransform(Math.log(scx) - scxlogcx / scx, this.base);
- }
-
- return entropy;
- } catch (ExecException ee) {
- throw ee;
- } catch (Exception e) {
- int errCode = 10082;
- String msg = "Error while computing average in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
- }
-
- }
- }
-
- static protected Tuple combine(DataBag values) throws ExecException {
- Tuple output = mTupleFactory.newTuple(2);
-
- boolean sawNonNull = false;
- double sumOfCxLogCx = 0;
- long sumOfCx = 0;
-
- for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
- Tuple t = it.next();
- Double scxlogcx = (Double)t.get(0);
- Long scx = (Long)t.get(1);
-
- if(scxlogcx != null && scx != null) {
- sumOfCxLogCx += scxlogcx.doubleValue();
- sumOfCx += scx.longValue();
- sawNonNull = true;
- }
- }
-
- if (sawNonNull) {
- output.set(0, sumOfCxLogCx);
- output.set(1, sumOfCx);
- } else {
- //emit null if there is no invalid input
- output.set(0, null);
- output.set(1, null);
- }
-
- return output;
- }
-
- /*
- * Accumulator implementation part
- */
-
- @Override
- public void accumulate(Tuple input) throws IOException
- {
- for (Tuple t : (DataBag) input.get(0)) {
- long cx = ((Number)(t.get(0))).longValue();
- this.streamEstimator.accumulate(cx);
- }
- }
-
- @Override
- public Double getValue()
- {
- return this.streamEstimator.getEntropy();
- }
-
- @Override
- public void cleanup()
- {
- this.streamEstimator.reset();
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try {
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.BAG)
- {
- throw new RuntimeException("Expected a BAG as input");
- }
-
- Schema inputBagSchema = inputFieldSchema.schema;
-
- if (inputBagSchema.getField(0).type != DataType.TUPLE)
- {
- throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
- DataType.findTypeName(inputBagSchema.getField(0).type)));
- }
-
- Schema tupleSchema = inputBagSchema.getField(0).schema;
-
- if(tupleSchema == null) {
- throw new RuntimeException("The tuple of input bag has no schema");
- }
-
- List<Schema.FieldSchema> fieldSchemaList = tupleSchema.getFields();
-
- if(fieldSchemaList == null || fieldSchemaList.size() != 1) {
- throw new RuntimeException("The field schema of the input tuple is null or its size is not 1");
- }
-
- if(fieldSchemaList.get(0).type != DataType.INTEGER &&
- fieldSchemaList.get(0).type != DataType.LONG )
- {
- String[] expectedTypes = new String[] {DataType.findTypeName(DataType.INTEGER),
- DataType.findTypeName(DataType.LONG)};
- throw new RuntimeException("Expect the type of the input tuple to be of (" +
- java.util.Arrays.toString(expectedTypes) + "), but instead found " +
- DataType.findTypeName(fieldSchemaList.get(0).type));
- }
-
- return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
- .getName()
- .toLowerCase(), input),
- DataType.DOUBLE));
- } catch (FrontendException e) {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/entropy/EmpiricalEntropyEstimator.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/entropy/EmpiricalEntropyEstimator.java b/src/java/datafu/pig/stats/entropy/EmpiricalEntropyEstimator.java
deleted file mode 100644
index 3f0acab..0000000
--- a/src/java/datafu/pig/stats/entropy/EmpiricalEntropyEstimator.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats.entropy;
-
-/*
- * This entropy estimator calculates the empirical entropy of given species
- * using the occurrence frequency of individual in the sample as an estimation of its probability.
- * The empirical estimator is also called the maximum likelihood estimator (MLE).
- * <p>
- * It applies the following formula to compute entropy:
- * <pre>
- * {@code
- * H(X) = log(N) - 1 / N * SUM(c(x) * log(c(x)) )
- * c(x) is the occurrence frequency of sample x, N is the sum of c(x)
- * }
- * </pre>
- * </p>
- * <p>
- * This entropy estimator is widely used when the number of species is known and small.
- * But it is biased since it does not cover the rare species with zero frequency in the sample.
- * </p>
- */
-class EmpiricalEntropyEstimator extends EntropyEstimator {
-
- //sum of frequency cx of all input samples
- private long N;
-
- //sum of cx * log(cx) of all input samples
- private double M;
-
- EmpiricalEntropyEstimator(String base) throws IllegalArgumentException {
- super(base);
- reset();
- }
-
- @Override
- public void accumulate(long cx) {
- if(cx > 0) {
- N += cx;
- M += cx * Math.log(cx);
- }
- }
-
- @Override
- public double getEntropy() {
- return N > 0 ? EntropyUtil.logTransform(Math.log(N) - M / N, super.base) : 0;
- }
-
- @Override
- public void reset(){
- N = 0;
- M = 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/entropy/Entropy.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/entropy/Entropy.java b/src/java/datafu/pig/stats/entropy/Entropy.java
deleted file mode 100644
index 9dfff1a..0000000
--- a/src/java/datafu/pig/stats/entropy/Entropy.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats.entropy;
-
-import java.io.IOException;
-
-import org.apache.pig.AccumulatorEvalFunc;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-
-
-/**
- * Calculate entropy H(X) of random variable X following entropy's
- * {@link <a href="http://en.wikipedia.org/wiki/Entropy_%28information_theory%29" target="_blank">wiki definition</a>}
- * <p>
- * This UDF's constructor takes 2 arguments.
- * </p>
- * <p>
- * The 1st argument, the type of entropy estimator algorithm we currently support, includes:
- * <ul>
- * <li>empirical (empirical entropy estimator)
- * <li>chaosh (Chao-Shen entropy estimator)
- * </ul>
- * </p>
- * <p>
- * The default estimation algorithm is empirical.
- * </p>
- * <p>
- * The 2nd argument, the logarithm base we currently support, includes:
- * </p>
- * <p>
- * <ul>
- * <li>log (use Euler's number as the logarithm base)
- * <li>log2 (use 2 as the logarithm base)
- * <li>log10 (use 10 as the logarithm base)
- * </ul>
- * </p>
- * <p>
- * The default logarithm base is log.
- * </p>
- * <p>
- * Note:
- * <ul>
- * <li>The input to this UDF must be a <b>sorted</b> bag of raw data tuples of X.
- * An exception will be thrown if the input bag is not sorted
- * <li>The returned entropy value is of double type.
- * </ul>
- * </p>
- * <p>
- * How to use:
- * </p>
- * <p>
- * This UDF calculates entropy from raw data tuples without the need to pre-compute per tuple occurrence frequency.
- * </p>
- * <p>
- * It could be used in a nested FOREACH after a GROUP BY, in which we sort the inner bag and use the sorted bag as this UDF's input.
- * </p>
- * Example:
- * <p>
- * <pre>
- * {@code
- * --calculate empirical entropy with Euler's number as the logarithm base
- * define Entropy datafu.pig.stats.entropy.Entropy();
- *
- * input = LOAD 'input' AS (grp: chararray, val: double);
- *
- * -- calculate the input's entropy in each group
- * input_group_g = GROUP input BY grp;
- * entropy_group = FOREACH input_group_g {
- * input_val = input.val;
- * input_ordered = ORDER input_val BY $0;
- * GENERATE FLATTEN(group) AS group, Entropy(input_ordered) AS entropy;
- * }
- * }
- * </pre>
- * </p>
- * @see CondEntropy
- * @see EmpiricalCountEntropy
- */
-public class Entropy extends AccumulatorEvalFunc<Double>
-{
- //last visited tuple
- private Tuple x;
-
- //number of occurrence of last visited tuple
- private long cx;
-
- //comparison result between the present tuple and the last visited tuple
- private int lastCmp;
-
- //entropy estimator that accumulates sample's occurrence frequency to
- //calculates the actual entropy
- private EntropyEstimator estimator;
-
- public Entropy() throws ExecException
- {
- this(EntropyEstimator.EMPIRICAL_ESTIMATOR);
- }
-
- public Entropy(String type) throws ExecException
- {
- this(type, EntropyUtil.LOG);
- }
-
- public Entropy(String type, String base) throws ExecException
- {
- try {
- this.estimator = EntropyEstimator.createEstimator(type, base);
- } catch (IllegalArgumentException ex) {
- throw new ExecException(
- String.format("Fail to initialize StreamingEntropy with entropy estimator of type (%s), base: (%s), exception: (%s)",
- type, base, ex)
- );
- }
- cleanup();
- }
-
- /*
- * Accumulate occurrence frequency of each tuple as we stream through the input bag
- */
- @Override
- public void accumulate(Tuple input) throws IOException
- {
- for (Tuple t : (DataBag) input.get(0)) {
-
- if (this.x != null)
- {
- int cmp = t.compareTo(this.x);
-
- //check if the comparison result is different from previous compare result
- if ((cmp < 0 && this.lastCmp > 0)
- || (cmp > 0 && this.lastCmp < 0)) {
- throw new ExecException("Out of order! previous tuple: " + this.x + ", present tuple: " + t
- + ", comparsion: " + cmp + ", previous comparsion: " + this.lastCmp);
- }
-
- if (cmp != 0) {
- //different tuple
- this.estimator.accumulate(this.cx);
- this.cx = 0;
- this.lastCmp = cmp;
- }
- }
-
- //set tuple t as the next tuple for comparison
- this.x = t;
-
- //accumulate cx
- this.cx++;
- }
- }
-
- @Override
- public Double getValue()
- {
- //do not miss the last tuple
- try {
- this.estimator.accumulate(this.cx);
- } catch (ExecException ex) {
- throw new RuntimeException("Error while accumulating sample frequency: " + ex);
- }
-
- return this.estimator.getEntropy();
- }
-
- @Override
- public void cleanup()
- {
- this.x = null;
- this.cx = 0;
- this.lastCmp = 0;
- this.estimator.reset();
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try {
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.BAG)
- {
- throw new RuntimeException("Expected a BAG as input");
- }
-
- Schema inputBagSchema = inputFieldSchema.schema;
-
- if (inputBagSchema.getField(0).type != DataType.TUPLE)
- {
- throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
- DataType.findTypeName(inputBagSchema.getField(0).type)));
- }
-
- return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
- .getName()
- .toLowerCase(), input),
- DataType.DOUBLE));
- } catch (FrontendException e) {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/entropy/EntropyEstimator.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/entropy/EntropyEstimator.java b/src/java/datafu/pig/stats/entropy/EntropyEstimator.java
deleted file mode 100644
index 336ef86..0000000
--- a/src/java/datafu/pig/stats/entropy/EntropyEstimator.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats.entropy;
-
-import org.apache.pig.backend.executionengine.ExecException;
-
-/*
-* Entropy estimator which exposes a unified interface to application
-* and hides the implementation details in its subclasses
-*/
-public abstract class EntropyEstimator {
-
- public static final String EMPIRICAL_ESTIMATOR = "empirical";
-
- public static final String CHAOSHEN_ESTIMATOR = "chaosh";
-
- /*
- * logarithm base
- * by default, we use Euler's number as the default logarithm base
- */
- protected String base;
-
- protected EntropyEstimator(String base) throws IllegalArgumentException {
- if(!EntropyUtil.isValidLogBase(base)) {
- throw new IllegalArgumentException("Invalid input logarithm base. " +
- "Please refer to StreamingEntropy's javadoc for supported logarithm base");
- }
- this.base = base;
- }
-
- /*
- * create estimator instance based on the input type
- * @param type type of entropy estimator
- * @param base logarithm base
- * if the method is not supported, a null estimator instance is returned
- * and the external application needs to handle this case
- */
- public static EntropyEstimator createEstimator(String type,
- String base) throws IllegalArgumentException
- {
- //empirical estimator
- if(EmpiricalEntropyEstimator.EMPIRICAL_ESTIMATOR.equalsIgnoreCase(type)) {
- return new EmpiricalEntropyEstimator(base);
- }
- //chao-shen estimator
- if(EmpiricalEntropyEstimator.CHAOSHEN_ESTIMATOR.equalsIgnoreCase(type)) {
- return new ChaoShenEntropyEstimator(base);
- }
- //unsupported estimator type
- throw new IllegalArgumentException("invalid input entropy estimator type. " +
- "Please refer to StreamingEntropy's javadoc for the supported estimator types");
- }
-
- /*
- * accumulate occurrence frequency from input stream of samples
- * @param cx the occurrence frequency of the last input sample
- */
- public abstract void accumulate(long cx) throws ExecException;
-
- /*
- * calculate the output entropy value
- * return entropy value as a double type number
- */
- public abstract double getEntropy();
-
- /*
- * cleanup and reset internal states
- */
- public abstract void reset();
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/entropy/EntropyUtil.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/entropy/EntropyUtil.java b/src/java/datafu/pig/stats/entropy/EntropyUtil.java
deleted file mode 100644
index 685f3e2..0000000
--- a/src/java/datafu/pig/stats/entropy/EntropyUtil.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats.entropy;
-
-public class EntropyUtil {
-
- public static final String LOG = "log";
-
- public static final String LOG2 = "log2";
-
- public static final String LOG10 = "log10";
-
- /*
- * Transform the input entropy to that in the input logarithm base
- * The input entropy is assumed to be calculated in the Euler's number logarithm base
- * */
- public static double logTransform(double h, String base) {
- if(LOG2.equalsIgnoreCase(base)) {
- return h / Math.log(2);
- }
-
- if(LOG10.equalsIgnoreCase(base)) {
- return h / Math.log(10);
- }
-
- return h;
- }
-
- public static boolean isValidLogBase(String base) {
- return LOG.equalsIgnoreCase(base) ||
- LOG2.equalsIgnoreCase(base) ||
- LOG10.equalsIgnoreCase(base);
- }
-}
[19/19] git commit: DATAFU-27 Migrate build system to Gradle
Posted by wv...@apache.org.
DATAFU-27 Migrate build system to Gradle
This migrates the build system from Ant to Gradle.
Signed-off-by: William Vaughan <wv...@linkedin.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-datafu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-datafu/commit/778bef1e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-datafu/tree/778bef1e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-datafu/diff/778bef1e
Branch: refs/heads/master
Commit: 778bef1eb64173891565a4c9ea15501619388aa6
Parents: 4aa2ef2
Author: Matt Hayes <mh...@linkedin.com>
Authored: Mon Mar 3 13:07:44 2014 -0800
Committer: William Vaughan <wv...@linkedin.com>
Committed: Mon Mar 3 19:35:49 2014 -0800
----------------------------------------------------------------------
.gitignore | 5 +-
.project | 18 -
HEADER | 16 +
README.md | 44 +-
build-plugin/.gitignore | 1 +
build-plugin/build.gradle | 34 +
.../multilinestring/EcjMultilineProcessor.java | 54 +
.../JavacMultilineProcessor.java | 49 +
.../adrianwalker/multilinestring/Multiline.java | 14 +
.../multilinestring/MultilineProcessor.java | 41 +
.../javax.annotation.processing.Processor | 1 +
build.gradle | 19 +
datafu-pig/.gitignore | 2 +
datafu-pig/build.gradle | 246 ++++
.../main/java/datafu/pig/bags/AppendToBag.java | 71 +
.../main/java/datafu/pig/bags/BagConcat.java | 187 +++
.../src/main/java/datafu/pig/bags/BagGroup.java | 182 +++
.../java/datafu/pig/bags/BagLeftOuterJoin.java | 268 ++++
.../src/main/java/datafu/pig/bags/BagSplit.java | 160 +++
.../main/java/datafu/pig/bags/CountEach.java | 166 +++
.../main/java/datafu/pig/bags/DistinctBy.java | 162 +++
.../java/datafu/pig/bags/EmptyBagToNull.java | 62 +
.../datafu/pig/bags/EmptyBagToNullFields.java | 95 ++
.../main/java/datafu/pig/bags/Enumerate.java | 153 +++
.../java/datafu/pig/bags/FirstTupleFromBag.java | 70 +
.../java/datafu/pig/bags/NullToEmptyBag.java | 53 +
.../main/java/datafu/pig/bags/PrependToBag.java | 79 ++
.../java/datafu/pig/bags/ReverseEnumerate.java | 142 ++
.../java/datafu/pig/bags/UnorderedPairs.java | 135 ++
.../main/java/datafu/pig/bags/package-info.java | 23 +
.../datafu/pig/geo/HaversineDistInMiles.java | 63 +
.../main/java/datafu/pig/geo/package-info.java | 23 +
.../src/main/java/datafu/pig/hash/MD5.java | 78 ++
.../src/main/java/datafu/pig/hash/SHA.java | 46 +
.../main/java/datafu/pig/hash/package-info.java | 23 +
.../java/datafu/pig/linkanalysis/PageRank.java | 482 +++++++
.../datafu/pig/linkanalysis/PageRankImpl.java | 571 ++++++++
.../pig/linkanalysis/ProgressIndicator.java | 25 +
.../datafu/pig/linkanalysis/package-info.java | 23 +
.../main/java/datafu/pig/random/RandInt.java | 65 +
.../main/java/datafu/pig/random/RandomUUID.java | 46 +
.../java/datafu/pig/random/package-info.java | 23 +
.../java/datafu/pig/sampling/Reservoir.java | 48 +
.../datafu/pig/sampling/ReservoirSample.java | 327 +++++
.../java/datafu/pig/sampling/SampleByKey.java | 130 ++
.../java/datafu/pig/sampling/ScoredTuple.java | 107 ++
.../datafu/pig/sampling/SimpleRandomSample.java | 489 +++++++
.../SimpleRandomSampleWithReplacementElect.java | 202 +++
.../SimpleRandomSampleWithReplacementVote.java | 281 ++++
.../pig/sampling/WeightedReservoirSample.java | 273 ++++
.../datafu/pig/sampling/WeightedSample.java | 198 +++
.../java/datafu/pig/sampling/package-info.java | 23 +
.../java/datafu/pig/sessions/SessionCount.java | 106 ++
.../java/datafu/pig/sessions/Sessionize.java | 185 +++
.../java/datafu/pig/sessions/package-info.java | 23 +
.../java/datafu/pig/sets/SetDifference.java | 262 ++++
.../main/java/datafu/pig/sets/SetIntersect.java | 132 ++
.../java/datafu/pig/sets/SetOperationsBase.java | 61 +
.../src/main/java/datafu/pig/sets/SetUnion.java | 79 ++
.../main/java/datafu/pig/sets/package-info.java | 23 +
.../main/java/datafu/pig/stats/DoubleVAR.java | 373 ++++++
.../main/java/datafu/pig/stats/FloatVAR.java | 375 ++++++
.../datafu/pig/stats/HyperLogLogPlusPlus.java | 116 ++
.../src/main/java/datafu/pig/stats/IntVAR.java | 372 ++++++
.../src/main/java/datafu/pig/stats/LongVAR.java | 373 ++++++
.../main/java/datafu/pig/stats/MarkovPairs.java | 150 +++
.../src/main/java/datafu/pig/stats/Median.java | 39 +
.../main/java/datafu/pig/stats/Quantile.java | 208 +++
.../java/datafu/pig/stats/QuantileUtil.java | 75 ++
.../java/datafu/pig/stats/StreamingMedian.java | 40 +
.../datafu/pig/stats/StreamingQuantile.java | 418 ++++++
.../src/main/java/datafu/pig/stats/VAR.java | 404 ++++++
.../java/datafu/pig/stats/WilsonBinConf.java | 142 ++
.../stats/entropy/ChaoShenEntropyEstimator.java | 230 ++++
.../datafu/pig/stats/entropy/CondEntropy.java | 263 ++++
.../stats/entropy/EmpiricalCountEntropy.java | 425 ++++++
.../entropy/EmpiricalEntropyEstimator.java | 71 +
.../java/datafu/pig/stats/entropy/Entropy.java | 224 ++++
.../pig/stats/entropy/EntropyEstimator.java | 87 ++
.../datafu/pig/stats/entropy/EntropyUtil.java | 51 +
.../java/datafu/pig/stats/package-info.java | 23 +
.../datafu/pig/text/opennlp/CachedFile.java | 41 +
.../java/datafu/pig/text/opennlp/POSTag.java | 177 +++
.../datafu/pig/text/opennlp/SentenceDetect.java | 122 ++
.../datafu/pig/text/opennlp/TokenizeME.java | 126 ++
.../datafu/pig/text/opennlp/TokenizeSimple.java | 101 ++
.../pig/text/opennlp/TokenizeWhitespace.java | 102 ++
.../java/datafu/pig/urls/UserAgentClassify.java | 42 +
.../main/java/datafu/pig/urls/package-info.java | 23 +
.../java/datafu/pig/util/AliasableEvalFunc.java | 268 ++++
.../src/main/java/datafu/pig/util/Assert.java | 32 +
.../main/java/datafu/pig/util/AssertUDF.java | 70 +
.../main/java/datafu/pig/util/BoolToInt.java | 31 +
.../src/main/java/datafu/pig/util/Coalesce.java | 188 +++
.../datafu/pig/util/ContextualEvalFunc.java | 83 ++
.../java/datafu/pig/util/DataFuException.java | 122 ++
.../java/datafu/pig/util/FieldNotFound.java | 42 +
.../src/main/java/datafu/pig/util/In.java | 32 +
.../src/main/java/datafu/pig/util/InUDF.java | 65 +
.../main/java/datafu/pig/util/IntToBool.java | 31 +
.../java/datafu/pig/util/SimpleEvalFunc.java | 221 +++
.../datafu/pig/util/TransposeTupleToBag.java | 113 ++
.../main/java/datafu/pig/util/package-info.java | 23 +
.../src/test/java/datafu/test/pig/PigTests.java | 260 ++++
.../java/datafu/test/pig/bags/BagTests.java | 1252 +++++++++++++++++
.../test/java/datafu/test/pig/geo/GeoTests.java | 91 ++
.../java/datafu/test/pig/hash/HashTests.java | 153 +++
.../pig/linkanalysis/PageRankImplTests.java | 286 ++++
.../test/pig/linkanalysis/PageRankTests.java | 153 +++
.../datafu/test/pig/random/NumberTests.java | 80 ++
.../java/datafu/test/pig/random/UUIDTests.java | 72 +
.../datafu/test/pig/sampling/SamplingTests.java | 523 ++++++++
.../pig/sampling/SimpleRandomSampleTest.java | 230 ++++
.../pig/sampling/SimpleRandomSampleTestOld.java | 176 +++
.../SimpleRandomSampleWithReplacementTest.java | 109 ++
.../WeightedReservoirSamplingTests.java | 373 ++++++
.../datafu/test/pig/sessions/SessionTests.java | 430 ++++++
.../java/datafu/test/pig/sets/SetTests.java | 330 +++++
.../datafu/test/pig/stats/EstimationTests.java | 73 +
.../datafu/test/pig/stats/MarkovPairTests.java | 143 ++
.../datafu/test/pig/stats/QuantileTests.java | 696 ++++++++++
.../java/datafu/test/pig/stats/VARTests.java | 521 ++++++++
.../test/pig/stats/WilsonBinConfTests.java | 98 ++
.../pig/stats/entropy/AbstractEntropyTests.java | 46 +
.../pig/stats/entropy/ChaoShenEntropyTests.java | 371 ++++++
.../pig/stats/entropy/CondEntropyTests.java | 407 ++++++
.../entropy/EmpiricalCountEntropyTests.java | 578 ++++++++
.../test/pig/stats/entropy/EntropyTests.java | 430 ++++++
.../java/datafu/test/pig/text/NLPTests.java | 190 +++
.../datafu/test/pig/urls/UserAgentTest.java | 71 +
.../datafu/test/pig/util/AliasEvalFuncTest.java | 95 ++
.../java/datafu/test/pig/util/AssertTests.java | 131 ++
.../datafu/test/pig/util/CoalesceTests.java | 532 ++++++++
.../test/java/datafu/test/pig/util/InTests.java | 133 ++
.../pig/util/IntBoolConversionPigTests.java | 107 ++
.../datafu/test/pig/util/TransposeTest.java | 116 ++
gradle.properties | 2 +
gradle/buildscript.gradle | 12 +
gradle/dependency-versions.gradle | 20 +
gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 49875 bytes
gradle/wrapper/gradle-wrapper.properties | 6 +
gradlew | 164 +++
.../multilinestring/EcjMultilineProcessor.java | 54 -
.../JavacMultilineProcessor.java | 49 -
.../adrianwalker/multilinestring/Multiline.java | 14 -
.../multilinestring/MultilineProcessor.java | 41 -
settings.gradle | 1 +
src/java/datafu/pig/bags/AppendToBag.java | 71 -
src/java/datafu/pig/bags/BagConcat.java | 187 ---
src/java/datafu/pig/bags/BagGroup.java | 182 ---
src/java/datafu/pig/bags/BagLeftOuterJoin.java | 268 ----
src/java/datafu/pig/bags/BagSplit.java | 160 ---
src/java/datafu/pig/bags/CountEach.java | 166 ---
src/java/datafu/pig/bags/DistinctBy.java | 162 ---
src/java/datafu/pig/bags/EmptyBagToNull.java | 62 -
.../datafu/pig/bags/EmptyBagToNullFields.java | 95 --
src/java/datafu/pig/bags/Enumerate.java | 153 ---
src/java/datafu/pig/bags/FirstTupleFromBag.java | 70 -
src/java/datafu/pig/bags/NullToEmptyBag.java | 53 -
src/java/datafu/pig/bags/PrependToBag.java | 79 --
src/java/datafu/pig/bags/ReverseEnumerate.java | 142 --
src/java/datafu/pig/bags/UnorderedPairs.java | 135 --
src/java/datafu/pig/bags/package-info.java | 23 -
.../datafu/pig/geo/HaversineDistInMiles.java | 63 -
src/java/datafu/pig/geo/package-info.java | 23 -
src/java/datafu/pig/hash/MD5.java | 78 --
src/java/datafu/pig/hash/SHA.java | 46 -
src/java/datafu/pig/hash/package-info.java | 23 -
src/java/datafu/pig/linkanalysis/PageRank.java | 482 -------
.../datafu/pig/linkanalysis/PageRankImpl.java | 571 --------
.../pig/linkanalysis/ProgressIndicator.java | 25 -
.../datafu/pig/linkanalysis/package-info.java | 23 -
src/java/datafu/pig/random/RandInt.java | 65 -
src/java/datafu/pig/random/RandomUUID.java | 46 -
src/java/datafu/pig/random/package-info.java | 23 -
src/java/datafu/pig/sampling/Reservoir.java | 48 -
.../datafu/pig/sampling/ReservoirSample.java | 327 -----
src/java/datafu/pig/sampling/SampleByKey.java | 130 --
src/java/datafu/pig/sampling/ScoredTuple.java | 107 --
.../datafu/pig/sampling/SimpleRandomSample.java | 489 -------
.../SimpleRandomSampleWithReplacementElect.java | 202 ---
.../SimpleRandomSampleWithReplacementVote.java | 281 ----
.../pig/sampling/WeightedReservoirSample.java | 273 ----
.../datafu/pig/sampling/WeightedSample.java | 198 ---
src/java/datafu/pig/sampling/package-info.java | 23 -
src/java/datafu/pig/sessions/SessionCount.java | 106 --
src/java/datafu/pig/sessions/Sessionize.java | 185 ---
src/java/datafu/pig/sessions/package-info.java | 23 -
src/java/datafu/pig/sets/SetDifference.java | 262 ----
src/java/datafu/pig/sets/SetIntersect.java | 132 --
src/java/datafu/pig/sets/SetOperationsBase.java | 61 -
src/java/datafu/pig/sets/SetUnion.java | 79 --
src/java/datafu/pig/sets/package-info.java | 23 -
src/java/datafu/pig/stats/DoubleVAR.java | 373 ------
src/java/datafu/pig/stats/FloatVAR.java | 375 ------
.../datafu/pig/stats/HyperLogLogPlusPlus.java | 116 --
src/java/datafu/pig/stats/IntVAR.java | 372 ------
src/java/datafu/pig/stats/LongVAR.java | 373 ------
src/java/datafu/pig/stats/MarkovPairs.java | 150 ---
src/java/datafu/pig/stats/Median.java | 39 -
src/java/datafu/pig/stats/Quantile.java | 208 ---
src/java/datafu/pig/stats/QuantileUtil.java | 75 --
src/java/datafu/pig/stats/StreamingMedian.java | 40 -
.../datafu/pig/stats/StreamingQuantile.java | 418 ------
src/java/datafu/pig/stats/VAR.java | 404 ------
src/java/datafu/pig/stats/WilsonBinConf.java | 142 --
.../stats/entropy/ChaoShenEntropyEstimator.java | 230 ----
.../datafu/pig/stats/entropy/CondEntropy.java | 263 ----
.../stats/entropy/EmpiricalCountEntropy.java | 425 ------
.../entropy/EmpiricalEntropyEstimator.java | 71 -
src/java/datafu/pig/stats/entropy/Entropy.java | 224 ----
.../pig/stats/entropy/EntropyEstimator.java | 87 --
.../datafu/pig/stats/entropy/EntropyUtil.java | 51 -
src/java/datafu/pig/stats/package-info.java | 23 -
.../datafu/pig/text/opennlp/CachedFile.java | 41 -
src/java/datafu/pig/text/opennlp/POSTag.java | 177 ---
.../datafu/pig/text/opennlp/SentenceDetect.java | 122 --
.../datafu/pig/text/opennlp/TokenizeME.java | 126 --
.../datafu/pig/text/opennlp/TokenizeSimple.java | 101 --
.../pig/text/opennlp/TokenizeWhitespace.java | 102 --
src/java/datafu/pig/urls/UserAgentClassify.java | 42 -
src/java/datafu/pig/urls/package-info.java | 23 -
src/java/datafu/pig/util/AliasableEvalFunc.java | 268 ----
src/java/datafu/pig/util/Assert.java | 32 -
src/java/datafu/pig/util/AssertUDF.java | 70 -
src/java/datafu/pig/util/BoolToInt.java | 31 -
src/java/datafu/pig/util/Coalesce.java | 188 ---
.../datafu/pig/util/ContextualEvalFunc.java | 83 --
src/java/datafu/pig/util/DataFuException.java | 122 --
src/java/datafu/pig/util/FieldNotFound.java | 42 -
src/java/datafu/pig/util/In.java | 32 -
src/java/datafu/pig/util/InUDF.java | 65 -
src/java/datafu/pig/util/IntToBool.java | 31 -
src/java/datafu/pig/util/SimpleEvalFunc.java | 221 ---
.../datafu/pig/util/TransposeTupleToBag.java | 113 --
src/java/datafu/pig/util/package-info.java | 23 -
test.sh | 2 +-
test/pig/datafu/test/pig/PigTests.java | 243 ----
test/pig/datafu/test/pig/bags/BagTests.java | 1254 ------------------
test/pig/datafu/test/pig/geo/GeoTests.java | 91 --
test/pig/datafu/test/pig/hash/HashTests.java | 153 ---
.../pig/linkanalysis/PageRankImplTests.java | 286 ----
.../test/pig/linkanalysis/PageRankTests.java | 153 ---
.../pig/datafu/test/pig/random/NumberTests.java | 80 --
test/pig/datafu/test/pig/random/UUIDTests.java | 73 -
.../datafu/test/pig/sampling/SamplingTests.java | 523 --------
.../pig/sampling/SimpleRandomSampleTest.java | 230 ----
.../pig/sampling/SimpleRandomSampleTestOld.java | 176 ---
.../SimpleRandomSampleWithReplacementTest.java | 109 --
.../WeightedReservoirSamplingTests.java | 373 ------
.../datafu/test/pig/sessions/SessionTests.java | 433 ------
test/pig/datafu/test/pig/sets/SetTests.java | 330 -----
.../datafu/test/pig/stats/EstimationTests.java | 73 -
.../datafu/test/pig/stats/MarkovPairTests.java | 143 --
.../datafu/test/pig/stats/QuantileTests.java | 696 ----------
test/pig/datafu/test/pig/stats/VARTests.java | 521 --------
.../test/pig/stats/WilsonBinConfTests.java | 98 --
.../pig/stats/entropy/AbstractEntropyTests.java | 46 -
.../pig/stats/entropy/ChaoShenEntropyTests.java | 373 ------
.../pig/stats/entropy/CondEntropyTests.java | 412 ------
.../entropy/EmpiricalCountEntropyTests.java | 585 --------
.../test/pig/stats/entropy/EntropyTests.java | 429 ------
test/pig/datafu/test/pig/text/NLPTests.java | 195 ---
.../pig/datafu/test/pig/urls/UserAgentTest.java | 71 -
.../datafu/test/pig/util/AliasEvalFuncTest.java | 95 --
test/pig/datafu/test/pig/util/AssertTests.java | 131 --
.../pig/datafu/test/pig/util/CoalesceTests.java | 532 --------
test/pig/datafu/test/pig/util/InTests.java | 133 --
.../pig/util/IntBoolConversionPigTests.java | 107 --
.../pig/datafu/test/pig/util/TransposeTest.java | 116 --
270 files changed, 22924 insertions(+), 22382 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 81e9ae8..d1b0869 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
/dist/
-/build/
+/.gradle/
+build/
annotation-plugin.jar
.factorypath
.classpath
@@ -15,6 +16,8 @@ pigunit-input-overriden.txt
*.asc
/bin/
.ant-targets-build.xml
+.project
+.settings/
/data/
*.iml
.idea
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/.project
----------------------------------------------------------------------
diff --git a/.project b/.project
deleted file mode 100644
index 022f4a7..0000000
--- a/.project
+++ /dev/null
@@ -1,18 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>datafu</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.apache.ivyde.eclipse.ivynature</nature>
- </natures>
-</projectDescription>
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/HEADER
----------------------------------------------------------------------
diff --git a/HEADER b/HEADER
new file mode 100644
index 0000000..60b675e
--- /dev/null
+++ b/HEADER
@@ -0,0 +1,16 @@
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 8b3726c..678fa4e 100644
--- a/README.md
+++ b/README.md
@@ -38,15 +38,55 @@ If you'd like to jump in and get started, check out the corresponding guides for
Bugs and feature requests can be filed [here](https://issues.apache.org/jira/browse/DATAFU). For other help please see the [discussion group](http://groups.google.com/group/datafu).
-## Building the Code
+## Developers
+
+### DataFu Pig
+
+#### Building the Code
The Apache DataFu Pig library can be built by running the command below. More information about working with the source
code can be found in the [DataFu Pig Contributing Guide](http://datafu.incubator.apache.org/docs/datafu/contributing.html).
```
-ant jar
+./gradlew assemble
+```
+
+The built JAR can be found under `datafu-pig/build/libs` by the name `datafu-pig-x.y.z.jar`, where x.y.z is the version.
+
+#### Generating Eclipse Files
+
+This command generates the eclipse project and classpath files:
+
+```
+./gradlew eclipse
+```
+
+To clean up the eclipse files:
+
+```
+./gradlew cleanEclipse
```
+#### Running the Tests
+
+To run all the tests:
+
+```
+./gradlew test
+```
+
+To run tests for a single class, use the `test.single` property. For example, to run only the QuantileTests:
+
+```
+/gradlew :datafu-pig:test -Dtest.single=QuantileTests
+```
+
+The tests can also be run from within eclipse.
+
+### DataFu Hourglass
+
+#### Building the Code
+
The Apache DataFu Pig library can be built by running the commands below. More information about working with the source
code can be found in the [DataFu Hourglass Contributing Guide](http://datafu.incubator.apache.org/docs/hourglass/contributing.html).
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/build-plugin/.gitignore
----------------------------------------------------------------------
diff --git a/build-plugin/.gitignore b/build-plugin/.gitignore
new file mode 100644
index 0000000..5e56e04
--- /dev/null
+++ b/build-plugin/.gitignore
@@ -0,0 +1 @@
+/bin
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/build-plugin/build.gradle
----------------------------------------------------------------------
diff --git a/build-plugin/build.gradle b/build-plugin/build.gradle
new file mode 100644
index 0000000..f16f2d2
--- /dev/null
+++ b/build-plugin/build.gradle
@@ -0,0 +1,34 @@
+apply plugin: 'java'
+
+repositories {
+ mavenCentral()
+}
+
+dependencies {
+ compile files("../staticlibs/org.eclipse.jdt.core_3.8.3.v20130121-145325.jar")
+ compile files("../staticlibs/org.eclipse.jdt.compiler.apt_1.0.500.v20120522-1651.jar")
+ compile files(getToolsJar())
+}
+
+def getToolsJar()
+{
+ def jar = files("${System.properties['java.home']}/../lib/tools.jar",
+ "${System.properties['java.home']}/lib/tools.jar",
+ "${System.properties['java.home']}/../Classes/classes.jar").find { it.exists() }
+
+ if (jar != null)
+ {
+ return jar
+ }
+
+ throw new GradleException("jdk tools jar not found")
+}
+
+cleanEclipse {
+ doLast {
+ delete ".apt_generated"
+ delete ".settings"
+ delete ".factorypath"
+ delete "bin"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/build-plugin/src/main/java/org/adrianwalker/multilinestring/EcjMultilineProcessor.java
----------------------------------------------------------------------
diff --git a/build-plugin/src/main/java/org/adrianwalker/multilinestring/EcjMultilineProcessor.java b/build-plugin/src/main/java/org/adrianwalker/multilinestring/EcjMultilineProcessor.java
new file mode 100644
index 0000000..7dbfe9a
--- /dev/null
+++ b/build-plugin/src/main/java/org/adrianwalker/multilinestring/EcjMultilineProcessor.java
@@ -0,0 +1,54 @@
+// Code from https://github.com/benelog/multiline.git
+// Based on Adrian Walker's blog post: http://www.adrianwalker.org/2011/12/java-multiline-string.html
+
+package org.adrianwalker.multilinestring;
+
+import java.util.Set;
+
+import javax.annotation.processing.AbstractProcessor;
+import javax.annotation.processing.ProcessingEnvironment;
+import javax.annotation.processing.RoundEnvironment;
+import javax.annotation.processing.SupportedAnnotationTypes;
+import javax.annotation.processing.SupportedSourceVersion;
+import javax.lang.model.SourceVersion;
+import javax.lang.model.element.Element;
+import javax.lang.model.element.TypeElement;
+import javax.lang.model.util.Elements;
+
+import org.eclipse.jdt.internal.compiler.apt.model.VariableElementImpl;
+import org.eclipse.jdt.internal.compiler.ast.FieldDeclaration;
+import org.eclipse.jdt.internal.compiler.ast.StringLiteral;
+import org.eclipse.jdt.internal.compiler.lookup.FieldBinding;
+
+import java.lang.reflect.Constructor;
+
+@SupportedAnnotationTypes({"org.adrianwalker.multilinestring.Multiline"})
+@SupportedSourceVersion(SourceVersion.RELEASE_6)
+public final class EcjMultilineProcessor extends AbstractProcessor {
+
+ private Elements elementUtils;
+
+ @Override
+ public void init(final ProcessingEnvironment procEnv) {
+ super.init(procEnv);
+ this.elementUtils = procEnv.getElementUtils();
+ }
+
+ @Override
+ public boolean process(final Set<? extends TypeElement> annotations, final RoundEnvironment roundEnv) {
+ Set<? extends Element> fields = roundEnv.getElementsAnnotatedWith(Multiline.class);
+
+ for (Element field : fields) {
+ String docComment = elementUtils.getDocComment(field);
+
+ if (null != docComment) {
+ VariableElementImpl fieldElem = (VariableElementImpl) field;
+ FieldBinding biding = (FieldBinding) fieldElem._binding;
+ FieldDeclaration decl = biding.sourceField();
+ StringLiteral string = new StringLiteral(docComment.toCharArray(), decl.sourceStart, decl.sourceEnd, decl.sourceStart);
+ decl.initialization = string;
+ }
+ }
+ return true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/build-plugin/src/main/java/org/adrianwalker/multilinestring/JavacMultilineProcessor.java
----------------------------------------------------------------------
diff --git a/build-plugin/src/main/java/org/adrianwalker/multilinestring/JavacMultilineProcessor.java b/build-plugin/src/main/java/org/adrianwalker/multilinestring/JavacMultilineProcessor.java
new file mode 100644
index 0000000..39aa24e
--- /dev/null
+++ b/build-plugin/src/main/java/org/adrianwalker/multilinestring/JavacMultilineProcessor.java
@@ -0,0 +1,49 @@
+// Code from https://github.com/benelog/multiline.git
+// Based on Adrian Walker's blog post: http://www.adrianwalker.org/2011/12/java-multiline-string.html
+
+package org.adrianwalker.multilinestring;
+
+import java.util.Set;
+
+import javax.annotation.processing.AbstractProcessor;
+import javax.annotation.processing.ProcessingEnvironment;
+import javax.annotation.processing.RoundEnvironment;
+import javax.annotation.processing.SupportedAnnotationTypes;
+import javax.annotation.processing.SupportedSourceVersion;
+import javax.lang.model.SourceVersion;
+import javax.lang.model.element.Element;
+import javax.lang.model.element.TypeElement;
+
+import com.sun.tools.javac.model.JavacElements;
+import com.sun.tools.javac.processing.JavacProcessingEnvironment;
+import com.sun.tools.javac.tree.JCTree.JCVariableDecl;
+import com.sun.tools.javac.tree.TreeMaker;
+
+@SupportedAnnotationTypes({"org.adrianwalker.multilinestring.Multiline"})
+@SupportedSourceVersion(SourceVersion.RELEASE_6)
+public final class JavacMultilineProcessor extends AbstractProcessor {
+
+ private JavacElements elementUtils;
+ private TreeMaker maker;
+
+ @Override
+ public void init(final ProcessingEnvironment procEnv) {
+ super.init(procEnv);
+ JavacProcessingEnvironment javacProcessingEnv = (JavacProcessingEnvironment) procEnv;
+ this.elementUtils = javacProcessingEnv.getElementUtils();
+ this.maker = TreeMaker.instance(javacProcessingEnv.getContext());
+ }
+
+ @Override
+ public boolean process(final Set<? extends TypeElement> annotations, final RoundEnvironment roundEnv) {
+ Set<? extends Element> fields = roundEnv.getElementsAnnotatedWith(Multiline.class);
+ for (Element field : fields) {
+ String docComment = elementUtils.getDocComment(field);
+ if (null != docComment) {
+ JCVariableDecl fieldNode = (JCVariableDecl) elementUtils.getTree(field);
+ fieldNode.init = maker.Literal(docComment);
+ }
+ }
+ return true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/build-plugin/src/main/java/org/adrianwalker/multilinestring/Multiline.java
----------------------------------------------------------------------
diff --git a/build-plugin/src/main/java/org/adrianwalker/multilinestring/Multiline.java b/build-plugin/src/main/java/org/adrianwalker/multilinestring/Multiline.java
new file mode 100644
index 0000000..c29d3ef
--- /dev/null
+++ b/build-plugin/src/main/java/org/adrianwalker/multilinestring/Multiline.java
@@ -0,0 +1,14 @@
+// Code from https://github.com/benelog/multiline.git
+// Based on Adrian Walker's blog post: http://www.adrianwalker.org/2011/12/java-multiline-string.html
+
+package org.adrianwalker.multilinestring;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target({ElementType.FIELD,ElementType.LOCAL_VARIABLE})
+@Retention(RetentionPolicy.SOURCE)
+public @interface Multiline {
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/build-plugin/src/main/java/org/adrianwalker/multilinestring/MultilineProcessor.java
----------------------------------------------------------------------
diff --git a/build-plugin/src/main/java/org/adrianwalker/multilinestring/MultilineProcessor.java b/build-plugin/src/main/java/org/adrianwalker/multilinestring/MultilineProcessor.java
new file mode 100644
index 0000000..9abdba5
--- /dev/null
+++ b/build-plugin/src/main/java/org/adrianwalker/multilinestring/MultilineProcessor.java
@@ -0,0 +1,41 @@
+// Code from https://github.com/benelog/multiline.git
+// Based on Adrian Walker's blog post: http://www.adrianwalker.org/2011/12/java-multiline-string.html
+
+package org.adrianwalker.multilinestring;
+
+import java.util.Set;
+
+import javax.annotation.processing.AbstractProcessor;
+import javax.annotation.processing.ProcessingEnvironment;
+import javax.annotation.processing.Processor;
+import javax.annotation.processing.RoundEnvironment;
+import javax.annotation.processing.SupportedAnnotationTypes;
+import javax.annotation.processing.SupportedSourceVersion;
+import javax.lang.model.SourceVersion;
+import javax.lang.model.element.TypeElement;
+
+@SupportedAnnotationTypes({"org.adrianwalker.multilinestring.Multiline"})
+@SupportedSourceVersion(SourceVersion.RELEASE_6)
+public final class MultilineProcessor extends AbstractProcessor {
+ private Processor delegator = null;
+
+ @Override
+ public void init(final ProcessingEnvironment procEnv) {
+ super.init(procEnv);
+ String envClassName = procEnv.getClass().getName();
+ if (envClassName.contains("com.sun.tools")) {
+ delegator = new JavacMultilineProcessor();
+ } else {
+ delegator = new EcjMultilineProcessor();
+ }
+ delegator.init(procEnv);
+ }
+
+ @Override
+ public boolean process(final Set<? extends TypeElement> annotations, final RoundEnvironment roundEnv) {
+ if (delegator == null ) {
+ return true;
+ }
+ return delegator.process(annotations, roundEnv);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/build-plugin/src/main/resources/META-INF/services/javax.annotation.processing.Processor
----------------------------------------------------------------------
diff --git a/build-plugin/src/main/resources/META-INF/services/javax.annotation.processing.Processor b/build-plugin/src/main/resources/META-INF/services/javax.annotation.processing.Processor
new file mode 100644
index 0000000..47e7137
--- /dev/null
+++ b/build-plugin/src/main/resources/META-INF/services/javax.annotation.processing.Processor
@@ -0,0 +1 @@
+org.adrianwalker.multilinestring.MultilineProcessor
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
new file mode 100644
index 0000000..9ed6eb1
--- /dev/null
+++ b/build.gradle
@@ -0,0 +1,19 @@
+buildscript {
+ repositories {
+ mavenCentral()
+ }
+
+ apply from: file('gradle/buildscript.gradle'), to: buildscript
+}
+
+allprojects {
+ repositories {
+ mavenCentral()
+ }
+}
+
+apply from: file("gradle/dependency-versions.gradle")
+
+allprojects {
+ apply plugin: 'eclipse'
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/.gitignore
----------------------------------------------------------------------
diff --git a/datafu-pig/.gitignore b/datafu-pig/.gitignore
new file mode 100644
index 0000000..634b0a2
--- /dev/null
+++ b/datafu-pig/.gitignore
@@ -0,0 +1,2 @@
+/bin
+/data
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/build.gradle
----------------------------------------------------------------------
diff --git a/datafu-pig/build.gradle b/datafu-pig/build.gradle
new file mode 100644
index 0000000..e21a5b1
--- /dev/null
+++ b/datafu-pig/build.gradle
@@ -0,0 +1,246 @@
+apply plugin: 'java'
+apply plugin: 'license'
+apply plugin: 'gradle-autojar'
+apply plugin: 'maven-publish'
+apply plugin: 'download-task'
+
+import groovy.xml.MarkupBuilder
+
+buildscript {
+ repositories {
+ mavenCentral()
+ }
+ dependencies {
+ classpath 'com.github.rholder:gradle-autojar:1.0.1'
+ classpath 'de.undercouch:gradle-download-task:0.5'
+ }
+}
+
+// the autojarred configuration includes all JARs that will be included
+// in the final JAR via autojar
+configurations.create('autojarred')
+
+configurations.create('jarjar')
+
+configurations {
+ compile {
+ extendsFrom autojarred
+ }
+}
+
+eclipse {
+ jdt {
+ file {
+ withProperties { properties ->
+ // set up annotation processing, which we use so we can have multi-line strings in our tests
+ properties.setProperty("org.eclipse.jdt.core.compiler.processAnnotations", "enabled")
+ }
+ }
+ }
+}
+
+// need to assemble the build plugin for eclipse since tests use the annotation processor defined there
+tasks.eclipse.dependsOn(':build-plugin:assemble')
+
+// more annotation processor setup
+eclipseJdt {
+ doFirst {
+ def f = file(".factorypath")
+ def w = new FileWriter(f)
+ def xml = new MarkupBuilder(w)
+ xml.setDoubleQuotes(true)
+ xml."factorypath"() {
+ "factorypathentry" (
+ kind: "EXTJAR",
+ id: configurations.testCompile.find {
+ it.name.startsWith("build-plugin")
+ },
+ enabled: true,
+ runInBatchMode: false
+ )
+ }
+ w.close()
+ }
+}
+
+cleanEclipse {
+ doLast {
+ delete ".apt_generated"
+ delete ".settings"
+ delete ".factorypath"
+ delete "bin"
+ }
+}
+
+jar
+{
+ // initial jar only includes the main classes of datafu, not the dependencies.
+ // this is not the one we'll publish.
+ classifier = "core"
+}
+
+task sourcesJar(type: Jar) {
+ description 'Creates the sources jar'
+
+ classifier = 'sources'
+ from sourceSets.main.allJava
+}
+
+task javadocJar(type: Jar, dependsOn: javadoc) {
+ description 'Creates the javadoc jar'
+
+ classifier = 'javadoc'
+ from javadoc.destinationDir
+}
+
+ext
+{
+ autojarBuildDir = tasks.jar.destinationDir
+}
+
+task jarWithDependencies(type: Autojar) {
+ description 'Creates a jar that includes the dependencies (under their own namespaces)'
+ autojarFiles = [tasks.jar.getArchivePath().absoluteFile]
+ targetConfiguration = configurations.autojarred
+ autojarExtra = '-baeq'
+}
+
+task jarWithDependenciesNamespaced(dependsOn: jarWithDependencies) {
+ description 'Creates the jar that includes dependencies (under a datafu namespace)'
+
+ outputFile = file(tasks.jar.getArchivePath().absoluteFile.toString().replace("-core","-jarjar"))
+
+ doLast {
+ project.ant {
+ taskdef name: "jarjar", classname: "com.tonicsystems.jarjar.JarJarTask", classpath: configurations.jarjar.asPath
+ jarjar(jarfile: outputFile, filesetmanifest: "merge") {
+ zipfileset(src: tasks.jarWithDependencies.autojarOutput)
+ rule pattern: "it.unimi.dsi.fastutil.**", result: "datafu.it.unimi.dsi.fastutil.@1"
+ rule pattern: "org.apache.commons.math.**", result: "datafu.org.apache.commons.math.@1"
+ rule pattern: "com.clearspring.analytics.**", result: "datafu.com.clearspring.analytics.@1"
+ rule pattern: "com.google.common.**", result: "datafu.com.google.common.@1"
+ rule pattern: "opennlp.**", result: "datafu.opennlp.@1"
+ }
+ }
+ }
+}
+
+task finalJar(type: Jar, dependsOn: jarWithDependenciesNamespaced) {
+ description 'Creates the final jar'
+
+ from(zipTree(jarWithDependenciesNamespaced.outputFile))
+}
+
+// don't publish the core archive, as this doesn't have the dependencies
+configurations.archives.artifacts.removeAll { return it.classifier == "core"; }
+
+artifacts {
+ archives sourcesJar
+ archives javadocJar
+ archives finalJar
+}
+
+// Note: alternate way to publish: https://github.com/Netflix/gradle-template
+
+publishing {
+ publications {
+ mavenJava(MavenPublication) {
+ artifact sourcesJar
+ artifact javadocJar
+ artifact finalJar
+
+ pom.withXml {
+ asNode().appendNode("packaging","jar")
+ asNode().appendNode("name","Apache DataFu")
+ asNode().appendNode("description","A collection of user-defined functions for working with large-scale data in Hadoop and Pig.")
+ asNode().appendNode("url","http://datafu.incubator.apache.org/")
+
+ def licenseNode = asNode().appendNode("licenses").appendNode("license")
+ licenseNode.appendNode("name","The Apache Software License, Version 2.0")
+ licenseNode.appendNode("url","http://www.apache.org/licenses/LICENSE-2.0.txt")
+ licenseNode.appendNode("distribution","repo")
+
+ def dependenciesNode = asNode().appendNode("dependencies")
+ def dependency = dependenciesNode.appendNode("dependency")
+ dependency.appendNode("groupId","joda-time")
+ dependency.appendNode("artifactId","joda-time")
+ dependency.appendNode("version","$jodaTimeVersion")
+ }
+ }
+ }
+}
+
+// create tasks to automatically add the license header
+license {
+ header rootProject.file('HEADER')
+ skipExistingHeaders = true
+}
+
+dependencies {
+ // dependencies that are packaged into the jar using autojar
+ // autojar only includes what is needed
+ autojarred "it.unimi.dsi:fastutil:$fastutilVersion"
+ autojarred "org.apache.commons:commons-math:$commonsMathVersion"
+ autojarred "com.clearspring.analytics:stream:$streamVersion"
+ autojarred "com.google.guava:guava:$guavaVersion"
+ autojarred "org.apache.opennlp:opennlp-tools:$openNlpVersion"
+ autojarred "org.apache.opennlp:opennlp-uima:$openNlpVersion"
+ autojarred "org.apache.opennlp:opennlp-maxent:$openNlpMaxEntVersion"
+
+ // needed to run jarjar
+ jarjar "com.googlecode.jarjar:jarjar:1.3"
+
+ // not included in autojar because it's already a pig dependency and so
+ // should be available
+ compile "joda-time:joda-time:$jodaTimeVersion"
+
+ // needed for compilation only. obviously don't need to autojar this.
+ compile "org.apache.pig:pig:$pigVersion"
+ compile "org.apache.hadoop:hadoop-core:$hadoopVersion"
+
+ testCompile "org.apache.pig:pigunit:$pigVersion"
+ testCompile "log4j:log4j:$log4jVersion"
+ testCompile "jline:jline:$jlineVersion"
+ testCompile "org.antlr:antlr:$antlrVersion"
+ testCompile "commons-io:commons-io:$commonsIoVersion"
+
+ testCompile "org.testng:testng:$testngVersion"
+ testCompile project(":build-plugin")
+}
+
+compileTestJava.doFirst {
+ options.compilerArgs = ['-processor', 'org.adrianwalker.multilinestring.MultilineProcessor']
+}
+
+// open NLP models used for testing. these are not shipped with datafu.
+task downloadOpenNlpModels << {
+ download {
+ src 'http://opennlp.sourceforge.net/models-1.5/en-pos-maxent.bin'
+ dest file('data/en-pos-maxent.bin')
+ onlyIfNewer true
+ }
+ download {
+ src 'http://opennlp.sourceforge.net/models-1.5/en-sent.bin'
+ dest file('data/en-sent.bin')
+ onlyIfNewer true
+ }
+ download {
+ src 'http://opennlp.sourceforge.net/models-1.5/en-token.bin'
+ dest file('data/en-token.bin')
+ onlyIfNewer true
+ }
+}
+
+// download models so can test in eclipse or testng
+tasks.eclipse.dependsOn('downloadOpenNlpModels')
+tasks.test.dependsOn('downloadOpenNlpModels')
+
+test {
+ // enable TestNG support (default is JUnit)
+ useTestNG()
+
+ systemProperty 'datafu.jar.dir', file('build/libs')
+ systemProperty 'datafu.data.dir', file('data')
+
+ maxHeapSize = "2G"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/bags/AppendToBag.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/bags/AppendToBag.java b/datafu-pig/src/main/java/datafu/pig/bags/AppendToBag.java
new file mode 100644
index 0000000..55c9e76
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/bags/AppendToBag.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.bags;
+
+import java.io.IOException;
+
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+import datafu.pig.util.SimpleEvalFunc;
+
+/**
+ * Appends a tuple to a bag.
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * define AppendToBag datafu.pig.bags.AppendToBag();
+ *
+ * -- input:
+ * -- ({(1),(2),(3)},(4))
+ * -- ({(10),(20),(30),(40),(50)},(60))
+ * input = LOAD 'input' AS (B: bag{T: tuple(v:INT)}, T: tuple(v:INT));
+
+ * -- output:
+ * -- ({(1),(2),(3),(4)})
+ * -- ({(10),(20),(30),(40),(50),(60)})
+ * output = FOREACH input GENERATE AppendToBag(B,T) as B;
+ * }
+ * </pre>
+ */
+public class AppendToBag extends SimpleEvalFunc<DataBag>
+{
+ public DataBag call(DataBag inputBag, Tuple t) throws IOException
+ {
+ inputBag.add(t);
+ return inputBag;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try {
+ return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
+ input.getField(0).schema, DataType.BAG));
+ }
+ catch (FrontendException e) {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/bags/BagConcat.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/bags/BagConcat.java b/datafu-pig/src/main/java/datafu/pig/bags/BagConcat.java
new file mode 100644
index 0000000..290f44b
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/bags/BagConcat.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.bags;
+
+import java.io.IOException;
+
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
+
+/**
+ * Unions all input bags to produce a single bag containing all tuples.
+ * <p>
+ * This UDF accepts two forms of input:
+ * <ol>
+ * <li>a tuple of 2 or more elements where each element is a bag with the same schema</li>
+ * <li>a single bag where each element of that bag is a bag and all of these bags have the same schema</li>
+ * </ol>
+ * <p>
+ * Example 1:
+ * <pre>
+ * {@code
+ * define BagConcat datafu.pig.bags.BagConcat();
+ * -- This example illustrates the use on a tuple of bags
+ *
+ * -- input:
+ * -- ({(1),(2),(3)},{(3),(4),(5)})
+ * -- ({(20),(25)},{(40),(50)})
+ * input = LOAD 'input' AS (A: bag{T: tuple(v:INT)}, B: bag{T: tuple(v:INT)});
+ *
+ * -- output:
+ * -- ({(1),(2),(3),(3),(4),(5)})
+ * -- ({(20),(25),(40),(50)})
+ * output = FOREACH input GENERATE BagConcat(A,B);
+ * }
+ * </pre>
+ * <p>
+ * Example 2:
+ * <pre>
+ * {@code
+ * define BagConcat datafu.pig.bags.BagConcat();
+ * -- This example illustrates the use on a bag of bags
+ *
+ * -- input:
+ * -- ({({(1),(2),(3)}),({(3),(4),(5)})})
+ * -- ({({(20),(25)}),({(40),(50)})})
+ * input = LOAD 'input' AS (A: bag{T: tuple(bag{T2: tuple(v:INT)})});
+ *
+ * -- output:
+ * -- ({(1),(2),(3),(3),(4),(5)})
+ * -- ({(20),(25),(40),(50)})
+ * output = FOREACH input GENERATE BagConcat(A);
+ * }
+ * </pre>
+ *
+ * @author wvaughan
+ *
+ */
+public class BagConcat extends EvalFunc<DataBag>
+{
+
+ @Override
+ public DataBag exec(Tuple input) throws IOException
+ {
+ DataBag output = BagFactory.getInstance().newDefaultBag();
+ if (input.size() > 1) {
+ // tuple of bags
+ for (int i=0; i < input.size(); i++) {
+ Object o = input.get(i);
+ if (!(o instanceof DataBag)) {
+ throw new RuntimeException("Expected a TUPLE of BAGs as input");
+ }
+
+ DataBag inputBag = (DataBag) o;
+ for (Tuple elem : inputBag) {
+ output.add(elem);
+ }
+ }
+ } else {
+ // bag of bags
+ DataBag outerBag = (DataBag)input.get(0);
+ for (Tuple outerTuple : outerBag) {
+ DataBag innerBag = (DataBag)outerTuple.get(0);
+ for (Tuple innerTuple : innerBag) {
+ output.add(innerTuple);
+ }
+ }
+ }
+ return output;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try {
+ Schema outputBagTupleSchema = null;
+
+ if (input.size() == 0) {
+ throw new RuntimeException("Expected input tuple to contain fields. Got none.");
+ }
+ // determine whether the input is a tuple of bags or a bag of bags
+ if (input.size() != 1) {
+ // tuple of bags
+
+ // verify that each element in the input is a bag
+ for (FieldSchema fieldSchema : input.getFields()) {
+ if (fieldSchema.type != DataType.BAG) {
+ throwBadTypeError("Expected a TUPLE of BAGs as input. Got instead: %s in schema: %s", fieldSchema.type, input);
+ }
+ if (fieldSchema.schema == null) {
+ throwBadSchemaError(fieldSchema.alias, input);
+ }
+ }
+
+ outputBagTupleSchema = input.getField(0).schema;
+ } else {
+ // bag of bags
+
+ // should only be a single element in the input and it should be a bag
+ FieldSchema fieldSchema = input.getField(0);
+ if (fieldSchema.type != DataType.BAG) {
+ throwBadTypeError("Expected a BAG of BAGs as input. Got instead: %s in schema: %s", fieldSchema.type, input);
+ }
+
+ // take the tuple schema from this outer bag
+ Schema outerBagTuple = input.getField(0).schema;
+ // verify that this tuple contains only a bag for each element
+ Schema outerBagSchema = outerBagTuple.getField(0).schema;
+ if (outerBagSchema.size() != 1) {
+ throw new RuntimeException(String.format("Expected outer bag to have only a single field. Got instead: %s",
+ outerBagSchema.prettyPrint()));
+ }
+ FieldSchema outerBagFieldSchema = outerBagSchema.getField(0);
+ if (outerBagFieldSchema.type != DataType.BAG) {
+ throwBadTypeError("Expected a BAG of BAGs as input. Got instead: %s", outerBagFieldSchema.type, outerBagTuple);
+ }
+
+ // take the schema of the inner tuple as the schema for the tuple in the output bag
+ FieldSchema innerTupleSchema = outerBagSchema.getField(0);
+ if (innerTupleSchema.schema == null) {
+ throwBadSchemaError(innerTupleSchema.alias, outerBagSchema);
+ }
+ outputBagTupleSchema = innerTupleSchema.schema;
+ }
+
+ // return the correct schema
+ return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
+ outputBagTupleSchema, DataType.BAG));
+ }
+ catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void throwBadSchemaError(String alias, Schema schema) throws FrontendException
+ {
+ throw new FrontendException(String.format("Expected non-null schema for all bags. Got null on field %s, in: %s",
+ alias, schema.prettyPrint()));
+ }
+
+ private void throwBadTypeError(String expectedMessage, byte actualType, Schema schema) throws FrontendException
+ {
+ throw new FrontendException(String.format(expectedMessage, DataType.findTypeName(actualType), schema.prettyPrint()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/bags/BagGroup.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/bags/BagGroup.java b/datafu-pig/src/main/java/datafu/pig/bags/BagGroup.java
new file mode 100644
index 0000000..7b6c2e2
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/bags/BagGroup.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.bags;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
+
+import datafu.pig.util.AliasableEvalFunc;
+
+/**
+ * Performs an in-memory group operation on a bag. The first argument is the bag.
+ * The second argument is a projection of that bag to the group keys.
+ *
+ * <p>
+ * Example:
+ * <code>
+ * define BagGroup datafu.pig.bags.BagGroup();
+ *
+ * data = LOAD 'input' AS (input_bag: bag {T: tuple(k: int, v: chararray)});
+ * -- ({(1,A),(1,B),(2,A),(2,B),(2,C),(3,A)})
+ *
+ * data2 = FOREACH data GENERATE BagGroup(input_bag, input_bag.(k)) as grouped;
+ * -- data2: {grouped: {(group: int,input_bag: {T: (k: int,v: chararray)})}}
+ * -- ({(1,{(1,A),(1,B)}),(2,{(2,A),(2,B),(2,C)}),(3,{(3,A)})})
+ * </code>
+ * </p>
+ *
+ * @author wvaughan
+ *
+ */
+public class BagGroup extends AliasableEvalFunc<DataBag>
+{
+ private final String FIELD_NAMES_PROPERTY = "FIELD_NAMES";
+ private List<String> fieldNames;
+
+ @Override
+ public Schema getOutputSchema(Schema input)
+ {
+ try {
+ if (input.size() != 2) {
+ throw new RuntimeException(String.format("Expected input of format (BAG, PROJECTED_BAG...). Got %d field.", input.size()));
+ }
+ // Expect the first field to be a bag
+ FieldSchema bagFieldSchema = input.getField(0);
+ if (bagFieldSchema.type != DataType.BAG) {
+ throw new RuntimeException(String.format("Expected input of format (BAG, PROJECTED_BAG...). Got %s as first field.", DataType.findTypeName(bagFieldSchema.type)));
+ }
+ // Expect the second fields to be a projection of the bag
+ FieldSchema projectedBagFieldSchema = input.getField(1);
+ if (projectedBagFieldSchema.type != DataType.BAG) {
+ throw new RuntimeException(String.format("Expected input of format (BAG, PROJECTED_BAG...). Got %s as second field.", DataType.findTypeName(projectedBagFieldSchema.type)));
+ }
+
+ String bagName = bagFieldSchema.alias;
+ // handle named tuples
+ if (bagFieldSchema.schema.size() == 1) {
+ FieldSchema bagTupleFieldSchema = bagFieldSchema.schema.getField(0);
+ if (bagTupleFieldSchema.type == DataType.TUPLE && bagTupleFieldSchema.alias != null) {
+ bagName = getPrefixedAliasName(bagName, bagTupleFieldSchema.alias);
+ }
+ }
+ if (projectedBagFieldSchema.schema.size() == 1) {
+ FieldSchema projectedBagTupleFieldSchema = projectedBagFieldSchema.schema.getField(0);
+ if (projectedBagTupleFieldSchema.type == DataType.TUPLE && projectedBagTupleFieldSchema.schema != null) {
+ projectedBagFieldSchema = projectedBagTupleFieldSchema;
+ }
+ }
+
+ // create the output schema for the 'group'
+ // store the field names for the group keys
+ Schema groupTupleSchema = new Schema();
+ fieldNames = new ArrayList<String>(projectedBagFieldSchema.schema.size());
+ for (int i=0; i<projectedBagFieldSchema.schema.size(); i++) {
+ FieldSchema fieldSchema = projectedBagFieldSchema.schema.getField(i);
+ String fieldName = fieldSchema.alias;
+ fieldNames.add(getPrefixedAliasName(bagName, fieldName));
+ groupTupleSchema.add(new FieldSchema(fieldSchema.alias, fieldSchema.type));
+ }
+ getInstanceProperties().put(FIELD_NAMES_PROPERTY, fieldNames);
+
+ Schema outputTupleSchema = new Schema();
+ if (projectedBagFieldSchema.schema.size() > 1) {
+ // multiple group keys
+ outputTupleSchema.add(new FieldSchema("group", groupTupleSchema, DataType.TUPLE));
+ } else {
+ // single group key
+ outputTupleSchema.add(new FieldSchema("group", groupTupleSchema.getField(0).type));
+ }
+ outputTupleSchema.add(bagFieldSchema);
+
+ return new Schema(new Schema.FieldSchema(
+ getSchemaName(this.getClass().getName().toLowerCase(), input),
+ outputTupleSchema,
+ DataType.BAG));
+ } catch (FrontendException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ Map<Tuple, List<Tuple>> groups = new HashMap<Tuple, List<Tuple>>();
+ TupleFactory tupleFactory = TupleFactory.getInstance();
+ BagFactory bagFactory = BagFactory.getInstance();
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public DataBag exec(Tuple input) throws IOException
+ {
+ fieldNames = (List<String>)getInstanceProperties().get(FIELD_NAMES_PROPERTY);
+
+ DataBag inputBag = (DataBag)input.get(0);
+
+ for (Tuple tuple : inputBag) {
+ Tuple key = extractKey(tuple);
+ addGroup(key, tuple);
+ }
+
+ DataBag outputBag = bagFactory.newDefaultBag();
+ for (Tuple key : groups.keySet()) {
+ Tuple outputTuple = tupleFactory.newTuple();
+ if (fieldNames.size() > 1) {
+ outputTuple.append(key);
+ } else {
+ outputTuple.append(key.get(0));
+ }
+ DataBag groupBag = bagFactory.newDefaultBag();
+ for (Tuple groupedTuple : groups.get(key)) {
+ groupBag.add(groupedTuple);
+ }
+ outputTuple.append(groupBag);
+ outputBag.add(outputTuple);
+ }
+
+ return outputBag;
+ }
+
+ private Tuple extractKey(Tuple tuple) throws ExecException {
+ Tuple key = tupleFactory.newTuple();
+ for (String field : fieldNames) {
+ key.append(getObject(tuple, field));
+ }
+ return key;
+ }
+
+ private void addGroup(Tuple key, Tuple value) {
+ if (!groups.containsKey(key)) {
+ groups.put(key, new LinkedList<Tuple>());
+ }
+ groups.get(key).add(value);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/bags/BagLeftOuterJoin.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/bags/BagLeftOuterJoin.java b/datafu-pig/src/main/java/datafu/pig/bags/BagLeftOuterJoin.java
new file mode 100644
index 0000000..ba6bc11
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/bags/BagLeftOuterJoin.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.bags;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
+
+import datafu.pig.util.AliasableEvalFunc;
+
+/**
+ * Performs an in-memory left outer join across multiple bags.
+ *
+ * <p>
+ * The format for invocation is BagLeftOuterJoin(bag, 'key',....).
+ * This UDF expects that all bags are non-null and that there is a corresponding key for each bag.
+ * The <em>key</em> that is expected is the alias of the key inside of the preceding bag.
+ * </p>
+ *
+ * <p>
+ * Example:
+ * <code>
+ * define BagLeftOuterJoin datafu.pig.bags.BagLeftOuterJoin();
+ *
+ * -- describe data:
+ * -- data: {bag1: {(key1: chararray,value1: chararray)},bag2: {(key2: chararray,value2: int)}}
+ *
+ * bag_joined = FOREACH data GENERATE BagLeftOuterJoin(bag1, 'key1', bag2, 'key2') as joined;
+ *
+ * -- describe bag_joined:
+ * -- bag_joined: {joined: {(bag1::key1: chararray, bag1::value1: chararray, bag2::key2: chararray, bag2::value2: int)}}
+ * </code>
+ * </p>
+ *
+ * @author wvaughan
+ *
+ */
+public class BagLeftOuterJoin extends AliasableEvalFunc<DataBag>
+{
+
+ private static final String BAG_NAMES_PROPERTY = "BagLeftOuterJoin_BAG_NAMES";
+ private static final String BAG_NAME_TO_JOIN_PREFIX_PROPERTY = "BagLeftOuterJoin_BAG_NAME_TO_JOIN_PREFIX";
+ private static final String BAG_NAME_TO_SIZE_PROPERTY = "BagLeftOuterJoin_BAG_NAME_TO_SIZE_PROPERTY";
+
+ ArrayList<String> bagNames;
+ Map<String, String> bagNameToJoinKeyPrefix;
+ Map<String, Integer> bagNameToSize;
+
+ public BagLeftOuterJoin() {
+
+ }
+
+ @SuppressWarnings("unchecked")
+ private void retrieveContextValues()
+ {
+ Properties properties = getInstanceProperties();
+ bagNames = (ArrayList<String>) properties.get(BAG_NAMES_PROPERTY);
+ bagNameToJoinKeyPrefix = (Map<String, String>) properties.get(BAG_NAME_TO_JOIN_PREFIX_PROPERTY);
+ bagNameToSize = (Map<String, Integer>) properties.get(BAG_NAME_TO_SIZE_PROPERTY);
+ }
+
+ class JoinCollector
+ {
+ HashMap<Object, List<Tuple>> joinData;
+
+ public void printJoinData() throws ExecException {
+ printData(joinData);
+ }
+
+ public void printData(HashMap<Object, List<Tuple>> data) throws ExecException {
+ for (Object o : data.keySet()) {
+ System.out.println(o);
+ for (Tuple t : data.get(o)) {
+ System.out.println("\t"+t.toDelimitedString(", "));
+ }
+ }
+ }
+
+ public HashMap<Object, List<Tuple>> groupTuples(Iterable<Tuple> tuples, String keyName) throws ExecException {
+ HashMap<Object, List<Tuple>> group = new HashMap<Object, List<Tuple>>();
+ for (Tuple tuple : tuples) {
+ Object key = getObject(tuple, keyName);
+ if (!group.containsKey(key)) {
+ group.put(key, new LinkedList<Tuple>());
+ }
+ group.get(key).add(tuple);
+ }
+ return group;
+ }
+
+ public HashMap<Object, List<Tuple>> insertNullTuples(HashMap<Object, List<Tuple>> groupedData, int tupleSize) throws ExecException {
+ Tuple nullTuple = TupleFactory.getInstance().newTuple(tupleSize);
+ for (int i=0; i<tupleSize; i++) {
+ nullTuple.set(i, null);
+ }
+ for (Object key : joinData.keySet()) {
+ if (!groupedData.containsKey(key)) {
+ groupedData.put(key, Collections.singletonList(nullTuple));
+ }
+ }
+ return groupedData;
+ }
+
+ public void joinTuples(Object key, List<Tuple> tuples) throws ExecException {
+ List<Tuple> currentTuples = joinData.get(key);
+ if (currentTuples != null) {
+ List<Tuple> newTuples = new LinkedList<Tuple>();
+ if (tuples != null) {
+ for (Tuple t1 : currentTuples) {
+ for (Tuple t2 : tuples) {
+ Tuple t = TupleFactory.getInstance().newTuple();
+ for (Object o : t1.getAll()) {
+ t.append(o);
+ }
+ for (Object o : t2.getAll()) {
+ t.append(o);
+ }
+ newTuples.add(t);
+ }
+ }
+ }
+ joinData.put(key, newTuples);
+ }
+ }
+
+ public HashMap<Object, List<Tuple>> getJoinData() {
+ return this.joinData;
+ }
+
+ public void setJoinData(HashMap<Object, List<Tuple>> joinData) {
+ this.joinData = joinData;
+ }
+ }
+
+ @Override
+ public DataBag exec(Tuple input) throws IOException
+ {
+ retrieveContextValues();
+
+ ArrayList<String> joinKeyNames = new ArrayList<String>();
+ for (int i = 1; i < input.size(); i += 2) {
+ joinKeyNames.add((String) input.get(i));
+ }
+
+ JoinCollector collector = new JoinCollector();
+ // the first bag is the outer bag
+ String leftBagName = bagNames.get(0);
+ DataBag leftBag = getBag(input, leftBagName);
+ String leftBagJoinKeyName = getPrefixedAliasName(bagNameToJoinKeyPrefix.get(leftBagName), joinKeyNames.get(0));
+ collector.setJoinData(collector.groupTuples(leftBag, leftBagJoinKeyName));
+ // now, for each additional bag, group up the tuples by the join key, then join them in
+ if (bagNames.size() > 1) {
+ for (int i = 1; i < bagNames.size(); i++) {
+ String bagName = bagNames.get(i);
+ DataBag bag = getBag(input, bagName);
+ String joinKeyName = getPrefixedAliasName(bagNameToJoinKeyPrefix.get(bagName), joinKeyNames.get(i));
+ int tupleSize = bagNameToSize.get(bagName);
+ if (bag == null) throw new IOException("Error in instance: "+getInstanceName()
+ + " with properties: " + getInstanceProperties()
+ + " and tuple: " + input.toDelimitedString(", ")
+ + " -- Expected bag, got null");
+ HashMap<Object, List<Tuple>> groupedData = collector.groupTuples(bag, joinKeyName);
+ // outer join, so go back in and add nulls;
+ groupedData = collector.insertNullTuples(groupedData, tupleSize);
+ for (Map.Entry<Object, List<Tuple>> entry : groupedData.entrySet()) {
+ collector.joinTuples(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+
+ // assemble output bag
+ DataBag outputBag = BagFactory.getInstance().newDefaultBag();
+ for (List<Tuple> tuples : collector.getJoinData().values()) {
+ for (Tuple tuple : tuples) {
+ outputBag.add(tuple);
+ }
+ }
+
+ return outputBag;
+ }
+
+ @Override
+ public Schema getOutputSchema(Schema input)
+ {
+ ArrayList<String> bagNames = new ArrayList<String>(input.size() / 2);
+ Map<String, String> bagNameToJoinPrefix = new HashMap<String, String>(input.size() / 2);
+ Map<String, Integer> bagNameToSize = new HashMap<String, Integer>(input.size() / 2);
+ Schema outputSchema = null;
+ Schema bagSchema = new Schema();
+ try {
+ int i = 0;
+ // all even fields should be bags, odd fields are key names
+ String bagName = null;
+ String tupleName = null;
+ for (FieldSchema outerField : input.getFields()) {
+ if (i++ % 2 == 1)
+ continue;
+ bagName = outerField.alias;
+ bagNames.add(bagName);
+ if (bagName == null)
+ bagName = "null";
+ if (outerField.schema == null)
+ throw new RuntimeException("Expected input format of (bag, 'field') pairs. "
+ +"Did not receive a bag at index: "+i+", alias: "+bagName+". "
+ +"Instead received type: "+DataType.findTypeName(outerField.type)+" in schema:"+input.toString());
+ FieldSchema tupleField = outerField.schema.getField(0);
+ tupleName = tupleField.alias;
+ bagNameToJoinPrefix.put(bagName, getPrefixedAliasName(outerField.alias, tupleName));
+ if (tupleField.schema == null) {
+ log.error(String.format("could not get schema for inner tuple %s in bag %s", tupleName, bagName));
+ } else {
+ bagNameToSize.put(bagName, tupleField.schema.size());
+ for (FieldSchema innerField : tupleField.schema.getFields()) {
+ String innerFieldName = innerField.alias;
+ if (innerFieldName == null)
+ innerFieldName = "null";
+ String outputFieldName = bagName + "::" + innerFieldName;
+ bagSchema.add(new FieldSchema(outputFieldName, innerField.type));
+ }
+ }
+ }
+ outputSchema = new Schema(new Schema.FieldSchema("joined", bagSchema, DataType.BAG));
+ log.debug("output schema: "+outputSchema.toString());
+ } catch (FrontendException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ Properties properties = getInstanceProperties();
+ properties.put(BAG_NAMES_PROPERTY, bagNames);
+ properties.put(BAG_NAME_TO_JOIN_PREFIX_PROPERTY, bagNameToJoinPrefix);
+ properties.put(BAG_NAME_TO_SIZE_PROPERTY, bagNameToSize);
+ return outputSchema;
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/bags/BagSplit.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/bags/BagSplit.java b/datafu-pig/src/main/java/datafu/pig/bags/BagSplit.java
new file mode 100644
index 0000000..74d39d0
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/bags/BagSplit.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.bags;
+
+import java.io.IOException;
+
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * Splits a bag of tuples into a bag of bags, where the inner bags collectively contain
+ * the tuples from the original bag. This can be used to split a bag into a set of smaller bags.
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * define BagSplit datafu.pig.bags.BagSplit();
+ *
+ * -- input:
+ * -- ({(1),(2),(3),(4),(5),(6),(7)})
+ * -- ({(1),(2),(3),(4),(5)})
+ * -- ({(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11)})
+ * input = LOAD 'input' AS (B:bag{T:tuple(val1:INT,val2:INT)});
+ *
+ * -- ouput:
+ * -- ({{(1),(2),(3),(4),(5)},{(6),(7)}})
+ * -- ({{(1),(2),(3),(4),(5)},{(6),(7),(8),(9),(10)},{(11)}})
+ * output = FOREACH input GENERATE BagSplit(5,B);
+ * }
+ * </pre>
+ */
+public class BagSplit extends EvalFunc<DataBag>
+{
+ private static final BagFactory bagFactory = BagFactory.getInstance();
+ private static final TupleFactory tupleFactory = TupleFactory.getInstance();
+
+ private final boolean appendBagNum;
+
+ public BagSplit()
+ {
+ this.appendBagNum = false;
+ }
+
+ public BagSplit(String appendBagNum)
+ {
+ this.appendBagNum = Boolean.parseBoolean(appendBagNum);
+ }
+
+ @Override
+ public DataBag exec(Tuple arg0) throws IOException
+ {
+ DataBag outputBag = bagFactory.newDefaultBag();
+
+ Integer maxSize = (Integer)arg0.get(0);
+
+ Object o = arg0.get(1);
+ if (!(o instanceof DataBag))
+ throw new RuntimeException("parameter must be a databag");
+
+ DataBag inputBag = (DataBag)o;
+
+ DataBag currentBag = null;
+
+ int count = 0;
+ int numBags = 0;
+ for (Tuple tuple : inputBag)
+ {
+ if (currentBag == null)
+ {
+ currentBag = bagFactory.newDefaultBag();
+ }
+
+ currentBag.add(tuple);
+ count++;
+
+ if (count >= maxSize)
+ {
+ Tuple newTuple = tupleFactory.newTuple();
+ newTuple.append(currentBag);
+
+ if (this.appendBagNum)
+ {
+ newTuple.append(numBags);
+ }
+
+ numBags++;
+
+ outputBag.add(newTuple);
+
+ count = 0;
+ currentBag = null;
+ }
+ }
+
+ if (currentBag != null)
+ {
+ Tuple newTuple = tupleFactory.newTuple();
+ newTuple.append(currentBag);
+ if (this.appendBagNum)
+ {
+ newTuple.append(numBags);
+ }
+ outputBag.add(newTuple);
+ }
+
+ return outputBag;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try {
+ if (input.getField(0).type != DataType.INTEGER)
+ {
+ throw new RuntimeException("Expected first argument to be an INTEGER");
+ }
+
+ if (input.getField(1).type != DataType.BAG)
+ {
+ throw new RuntimeException("Expected second argument to be a BAG");
+ }
+
+ Schema tupleSchema = new Schema();
+ tupleSchema.add(new Schema.FieldSchema("data", input.getField(1).schema.clone(), DataType.BAG));
+
+ if (this.appendBagNum)
+ {
+ tupleSchema.add(new Schema.FieldSchema("index", DataType.INTEGER));
+ }
+
+ return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
+ tupleSchema, DataType.BAG));
+ }
+ catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/bags/CountEach.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/bags/CountEach.java b/datafu-pig/src/main/java/datafu/pig/bags/CountEach.java
new file mode 100644
index 0000000..cfb0152
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/bags/CountEach.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.bags;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.pig.AccumulatorEvalFunc;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * Generates a count of the number of times each distinct tuple appears in a bag.
+ *
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * DEFINE CountEach datafu.pig.bags.CountEach();
+ * DEFINE CountEachFlatten datafu.pig.bags.CountEach('flatten');
+ *
+ * -- input:
+ * -- ({(A),(A),(C),(B)})
+ * input = LOAD 'input' AS (B: bag {T: tuple(alpha:CHARARRAY, numeric:INT)});
+ *
+ * -- output:
+ * -- {((A),2),((C),1),((B),1)}
+ * output = FOREACH input GENERATE CountEach(B);
+ *
+ * -- output_flatten:
+ * -- ({(A,2),(C,1),(B,1)})
+ * output_flatten = FOREACH input GENERATE CountEachFlatten(B);
+ * }
+ * </pre>
+ * </p>
+ */
+public class CountEach extends AccumulatorEvalFunc<DataBag>
+{
+ private boolean flatten = false;
+ private Map<Tuple, Integer> counts = new HashMap<Tuple, Integer>();
+
+ public CountEach() {
+
+ }
+
+ public CountEach(String arg) {
+ if (arg != null && arg.toLowerCase().equals("flatten")) {
+ flatten = true;
+ }
+ }
+
+ @Override
+ public void accumulate(Tuple input) throws IOException
+ {
+ DataBag inputBag = (DataBag)input.get(0);
+ if (inputBag == null) throw new IllegalArgumentException("Expected a bag, got null");
+
+ for (Tuple tuple : inputBag) {
+ if (!counts.containsKey(tuple)) {
+ counts.put(tuple, 0);
+ }
+ counts.put(tuple, counts.get(tuple)+1);
+ }
+ }
+
+ @Override
+ public DataBag getValue()
+ {
+ DataBag output = BagFactory.getInstance().newDefaultBag();
+ for (Tuple tuple : counts.keySet()) {
+ Tuple outputTuple = null;
+ Tuple innerTuple = TupleFactory.getInstance().newTuple(tuple.getAll());
+ if (flatten) {
+ innerTuple.append(counts.get(tuple));
+ outputTuple = innerTuple;
+ } else {
+ outputTuple = TupleFactory.getInstance().newTuple();
+ outputTuple.append(innerTuple);
+ outputTuple.append(counts.get(tuple));
+ }
+ output.add(outputTuple);
+ }
+
+ return output;
+ }
+
+ @Override
+ public void cleanup()
+ {
+ counts.clear();
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try {
+ if (input.size() != 1)
+ {
+ throw new RuntimeException("Expected input to have one field");
+ }
+
+ Schema.FieldSchema bagFieldSchema = input.getField(0);
+
+ if (bagFieldSchema.type != DataType.BAG)
+ {
+ throw new RuntimeException("Expected a BAG as input");
+ }
+
+ Schema inputBagSchema = bagFieldSchema.schema;
+
+ if (inputBagSchema.getField(0).type != DataType.TUPLE)
+ {
+ throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
+ DataType.findTypeName(inputBagSchema.getField(0).type)));
+ }
+
+ Schema inputTupleSchema = inputBagSchema.getField(0).schema;
+ if (inputTupleSchema == null) inputTupleSchema = new Schema();
+
+ Schema outputTupleSchema = null;
+
+ if (this.flatten) {
+ outputTupleSchema = inputTupleSchema.clone();
+ outputTupleSchema.add(new Schema.FieldSchema("count", DataType.INTEGER));
+ } else {
+ outputTupleSchema = new Schema();
+ outputTupleSchema.add(new Schema.FieldSchema("tuple_schema", inputTupleSchema.clone(), DataType.TUPLE));
+ outputTupleSchema.add(new Schema.FieldSchema("count", DataType.INTEGER));
+ }
+
+ return new Schema(new Schema.FieldSchema(
+ getSchemaName(this.getClass().getName().toLowerCase(), input),
+ outputTupleSchema,
+ DataType.BAG));
+ }
+ catch (CloneNotSupportedException e) {
+ throw new RuntimeException(e);
+ }
+ catch (FrontendException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/bags/DistinctBy.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/bags/DistinctBy.java b/datafu-pig/src/main/java/datafu/pig/bags/DistinctBy.java
new file mode 100644
index 0000000..a79e4de
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/bags/DistinctBy.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.bags;
+
+import java.io.IOException;
+import java.util.HashSet;
+
+import org.apache.pig.AccumulatorEvalFunc;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * Get distinct elements in a bag by a given set of field positions.
+ * The input and output schemas will be identical.
+ *
+ * The first tuple containing each distinct combination of these fields will be taken.
+ *
+ * This operation is order preserving. If both A and B appear in the output,
+ * and A appears before B in the input, then A will appear before B in the output.
+ *
+ * Example:
+ * <pre>
+ * {@code
+ * define DistinctBy datafu.pig.bags.DistinctBy('0');
+ *
+ * -- input:
+ * -- ({(a, 1),(a,1),(b, 2),(b,22),(c, 3),(d, 4)})
+ * input = LOAD 'input' AS (B: bag {T: tuple(alpha:CHARARRAY, numeric:INT)});
+ *
+ * output = FOREACH input GENERATE DistinctBy(B);
+ *
+ * -- output:
+ * -- ({(a,1),(b,2),(c,3),(d,4)})
+ * }
+ * </pre>
+ *
+ * @param map Any number of strings specifying field positions
+ */
+public class DistinctBy extends AccumulatorEvalFunc<DataBag>
+{
+ private HashSet<Integer> fields = new HashSet<Integer>();
+ private HashSet<Tuple> seen = new HashSet<Tuple>();
+ private DataBag outputBag;
+
+ public DistinctBy(String... fields)
+ {
+ for(String field : fields) {
+ this.fields.add(Integer.parseInt(field));
+ }
+ cleanup();
+ }
+
+ @Override
+ public void accumulate(Tuple input) throws IOException
+ {
+ if (input.size() != 1) {
+ throw new RuntimeException("Expected input to have only a single field");
+ }
+ if (input.getType(0) != DataType.BAG) {
+ throw new RuntimeException("Expected a BAG as input");
+ }
+
+ DataBag inputBag = (DataBag)input.get(0);
+ for (Tuple t : inputBag) {
+ Tuple distinctFieldTuple = getDistinctFieldTuple(t, this.fields);
+ if (!seen.contains(distinctFieldTuple)) {
+ outputBag.add(t);
+ seen.add(distinctFieldTuple);
+ }
+ }
+ }
+
+ @Override
+ public void cleanup()
+ {
+ seen.clear();
+ outputBag = BagFactory.getInstance().newDefaultBag();
+ }
+
+ @Override
+ public DataBag getValue()
+ {
+ return outputBag;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try {
+ if (input.size() != 1)
+ {
+ throw new RuntimeException("Expected input to have only a single field");
+ }
+
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.BAG)
+ {
+ throw new RuntimeException("Expected a BAG as input");
+ }
+
+ Schema inputBagSchema = inputFieldSchema.schema;
+
+ if (inputBagSchema.getField(0).type != DataType.TUPLE)
+ {
+ throw new RuntimeException(String.format("Expected input bag to contain a TUPLE, but instead found %s",
+ DataType.findTypeName(inputBagSchema.getField(0).type)));
+ }
+
+ Schema inputTupleSchema = inputBagSchema.getField(0).schema;
+
+ Schema outputTupleSchema = inputTupleSchema.clone();
+
+ return new Schema(new Schema.FieldSchema(
+ getSchemaName(this.getClass().getName().toLowerCase(), input),
+ outputTupleSchema,
+ DataType.BAG));
+ }
+ catch (CloneNotSupportedException e) {
+ throw new RuntimeException(e);
+ }
+ catch (FrontendException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private Tuple getDistinctFieldTuple(Tuple t, HashSet<Integer> distinctFieldPositions) throws ExecException {
+ Tuple fieldTuple = TupleFactory.getInstance().newTuple(distinctFieldPositions.size());
+ int idx = 0;
+ for(int i=0; i<t.size(); i++) {
+ if (distinctFieldPositions.contains(i)) {
+ fieldTuple.set(idx, t.get(i));
+ idx++;
+ }
+ }
+ return fieldTuple;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/bags/EmptyBagToNull.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/bags/EmptyBagToNull.java b/datafu-pig/src/main/java/datafu/pig/bags/EmptyBagToNull.java
new file mode 100644
index 0000000..5524ad5
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/bags/EmptyBagToNull.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.bags;
+
+import java.io.IOException;
+
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * Returns null if the input is an empty bag; otherwise,
+ * returns the input bag unchanged.
+ */
+public class EmptyBagToNull extends EvalFunc<DataBag>
+{
+ @Override
+ public DataBag exec(Tuple tuple) throws IOException
+ {
+ if (tuple.size() == 0 || tuple.get(0) == null)
+ return null;
+ Object o = tuple.get(0);
+ if (o instanceof DataBag)
+ {
+ DataBag bag = (DataBag)o;
+ if (bag.size() == 0)
+ {
+ return null;
+ }
+ else
+ {
+ return bag;
+ }
+ }
+ else
+ throw new IllegalArgumentException("expected a null or a bag");
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ return input;
+ }
+}
\ No newline at end of file
[02/19] DATAFU-27 Migrate build system to Gradle
Posted by wv...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/stats/VARTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/stats/VARTests.java b/test/pig/datafu/test/pig/stats/VARTests.java
deleted file mode 100644
index 776ed9e..0000000
--- a/test/pig/datafu/test/pig/stats/VARTests.java
+++ /dev/null
@@ -1,521 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.stats;
-
-import static org.testng.Assert.*;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.pigunit.PigTest;
-import org.junit.Assert;
-import org.testng.annotations.Test;
-
-import datafu.pig.stats.DoubleVAR;
-import datafu.pig.stats.FloatVAR;
-import datafu.pig.stats.IntVAR;
-import datafu.pig.stats.LongVAR;
-import datafu.pig.stats.VAR;
-import datafu.test.pig.PigTests;
-
-public class VARTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- define VAR datafu.pig.stats.VAR();
-
- data_in = LOAD 'input' as (val:$VAL_TYPE);
- data_out = GROUP data_in ALL;
- data_out = FOREACH data_out GENERATE VAR(data_in.val) AS variance;
-
- --describe data_out;
- STORE data_out into 'output';
- */
- @Multiline private String varTest;
-
- @Test
- public void varTestByteArray() throws Exception
- {
- PigTest test = createPigTestFromString(varTest,
- "VAL_TYPE=bytearray");
-
- String[] input = {"1","2","3","4","10","5","6","7","8","9"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(8.25)");
- }
-
- @Test
- public void varTestDouble() throws Exception
- {
- PigTest test = createPigTestFromString(varTest,
- "VAL_TYPE=double");
-
- String[] input = {"1.0","2.0","3.0","4.0","10.0","5.0","6.0","7.0","8.0","9.0"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(8.25)");
- }
-
- @Test
- public void varTestFloat() throws Exception
- {
- PigTest test = createPigTestFromString(varTest,
- "VAL_TYPE=float");
-
- String[] input = {"1.0","2.0","3.0","4.0","10.0","5.0","6.0","7.0","8.0","9.0"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(8.25)");
- }
-
- @Test
- public void varTestInteger() throws Exception
- {
- PigTest test = createPigTestFromString(varTest,
- "VAL_TYPE=int");
-
- String[] input = {"1","2","3","4","10","5","6","7","8","9"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(8.25)");
- }
-
- @Test
- public void varTestLong() throws Exception
- {
- PigTest test = createPigTestFromString(varTest,
- "VAL_TYPE=long");
-
- String[] input = {"1","2","3","4","10","5","6","7","8","9"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(8.25)");
- }
-
- @Test
- public void varTestOneData() throws Exception
- {
- PigTest test = createPigTestFromString(varTest,
- "VAL_TYPE=double");
-
- String[] input = {"5.0"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(0.0)");
- }
-
-
- @Test
- public void varTestZeroData() throws Exception
- {
- PigTest test = createPigTestFromString(varTest,
- "VAL_TYPE=long");
-
- String[] input = {};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),0);
- }
-
- @Test
- public void varTestNullEntry() throws Exception
- {
- PigTest test = createPigTestFromString(varTest,
- "VAL_TYPE=double");
-
- String[] input = {"1","2","3","4","10","5","6","7","8","9","null"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(8.25)");
- }
-
- @Test
- public void varTestNullEntries() throws Exception
- {
- PigTest test = createPigTestFromString(varTest,
- "VAL_TYPE=float");
-
- String[] input = {"1","2","3","4","10","5","6","7","8","9","null","null"};
- writeLinesToFile("input", input);
-
- test.runScript();
-
- List<Tuple> output = getLinesForAlias(test, "data_out", true);
-
- assertEquals(output.size(),1);
- assertEquals(output.get(0).toString(), "(8.25)");
- }
-
- @Test
- public void varExecTest() throws Exception {
- DoubleVAR var = new DoubleVAR();
-
- DataBag bag;
- Tuple input;
- Double result;
-
- bag = BagFactory.getInstance().newDefaultBag();
- for (int i=1; i<=1000; i++)
- {
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, (double)i);
- bag.add(t);
- }
-
- input = TupleFactory.getInstance().newTuple(1);
- input.set(0, bag);
-
- result = var.exec(input);
- Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
-
- // do it again to check cleanup
-
- bag = BagFactory.getInstance().newDefaultBag();
- for (int i=1; i<=2000; i++)
- {
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, (double)i);
- bag.add(t);
- }
-
- input = TupleFactory.getInstance().newTuple(1);
- input.set(0, bag);
-
- result = var.exec(input);
- Assert.assertTrue("Expected about 333333.2 but found " + result,Math.abs(333333.2 - result) < 1);
- }
-
- @Test
- public void varAccumulateTest() throws Exception {
- DoubleVAR var = new DoubleVAR();
-
- Double result;
-
- for (int i=1; i<=1000; i++)
- {
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, (double)i);
- DataBag bag = BagFactory.getInstance().newDefaultBag();
- bag.add(t);
- Tuple input = TupleFactory.getInstance().newTuple(1);
- input.set(0, bag);
- var.accumulate(input);
- }
-
- result = var.getValue();
- Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
-
- // do it again to check cleanup
- var.cleanup();
-
- for (int i=1; i<=2000; i++)
- {
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, (double)i);
- DataBag bag = BagFactory.getInstance().newDefaultBag();
- bag.add(t);
- Tuple input = TupleFactory.getInstance().newTuple(1);
- input.set(0, bag);
- var.accumulate(input);
- }
-
- result = var.getValue();
- Assert.assertTrue("Expected about 333333.2 but found " + result,Math.abs(333333.2 - result) < 1);
- }
-
- // make sure intermediate works, where initial just passes through a single tuple, and intermediate receives a large bag of the resulting tuples
- @Test
- public void varDoubleAlgebraicIntermediateTest() throws Exception {
- DoubleVAR.Initial initialVar = new DoubleVAR.Initial();
- DoubleVAR.Intermediate intermediateVar = new DoubleVAR.Intermediate();
- DoubleVAR.Final finalVar = new DoubleVAR.Final();
-
-
- DataBag intermediateBag = BagFactory.getInstance().newDefaultBag();
-
- for (int i=1; i<=1000; i++)
- {
- DataBag bag;
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, (double)i);
- bag = BagFactory.getInstance().newDefaultBag();
- bag.add(t);
- Tuple input = TupleFactory.getInstance().newTuple(bag);
- Tuple intermediateTuple = initialVar.exec(input);
- intermediateBag.add(intermediateTuple);
- }
-
- Tuple intermediateTuple = intermediateVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
- intermediateBag = BagFactory.getInstance().newDefaultBag(Arrays.asList(intermediateTuple));
- Double result = finalVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
-
- Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
- }
-
- // make sure final works, where initial just passes through a single tuple, intermediate does the same, and final receives the remainder
- @Test
- public void varDoubleAlgebraicFinalTest() throws Exception {
- DoubleVAR.Initial initialVar = new DoubleVAR.Initial();
- DoubleVAR.Intermediate intermediateVar = new DoubleVAR.Intermediate();
- DoubleVAR.Final finalVar = new DoubleVAR.Final();
-
- DataBag finalBag = BagFactory.getInstance().newDefaultBag();
-
- for (int i=1; i<=1000; i++)
- {
- DataBag bag;
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, (double)i);
- bag = BagFactory.getInstance().newDefaultBag();
- bag.add(t);
- Tuple input = TupleFactory.getInstance().newTuple(bag);
- Tuple intermediateTuple = initialVar.exec(input);
- DataBag intermediateBag = BagFactory.getInstance().newDefaultBag();
- intermediateBag.add(intermediateTuple);
- intermediateTuple = intermediateVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
- finalBag.add(intermediateTuple);
- }
-
- Double result = finalVar.exec(TupleFactory.getInstance().newTuple(finalBag));
-
- Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
- }
-
- // make sure intermediate works, where initial just passes through a single tuple, and intermediate receives a large bag of the resulting tuples
- @Test
- public void varFloatAlgebraicIntermediateTest() throws Exception {
- FloatVAR.Initial initialVar = new FloatVAR.Initial();
- FloatVAR.Intermediate intermediateVar = new FloatVAR.Intermediate();
- FloatVAR.Final finalVar = new FloatVAR.Final();
-
-
- DataBag intermediateBag = BagFactory.getInstance().newDefaultBag();
-
- for (int i=1; i<=1000; i++)
- {
- DataBag bag;
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, (float)i);
- bag = BagFactory.getInstance().newDefaultBag();
- bag.add(t);
- Tuple input = TupleFactory.getInstance().newTuple(bag);
- Tuple intermediateTuple = initialVar.exec(input);
- intermediateBag.add(intermediateTuple);
- }
-
- Tuple intermediateTuple = intermediateVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
- intermediateBag = BagFactory.getInstance().newDefaultBag(Arrays.asList(intermediateTuple));
- Double result = finalVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
-
- Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
- }
-
- // make sure final works, where initial just passes through a single tuple, intermediate does the same, and final receives the remainder
- @Test
- public void varFloatAlgebraicFinalTest() throws Exception {
- FloatVAR.Initial initialVar = new FloatVAR.Initial();
- FloatVAR.Intermediate intermediateVar = new FloatVAR.Intermediate();
- FloatVAR.Final finalVar = new FloatVAR.Final();
-
- DataBag finalBag = BagFactory.getInstance().newDefaultBag();
-
- for (int i=1; i<=1000; i++)
- {
- DataBag bag;
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, (float)i);
- bag = BagFactory.getInstance().newDefaultBag();
- bag.add(t);
- Tuple input = TupleFactory.getInstance().newTuple(bag);
- Tuple intermediateTuple = initialVar.exec(input);
- DataBag intermediateBag = BagFactory.getInstance().newDefaultBag();
- intermediateBag.add(intermediateTuple);
- intermediateTuple = intermediateVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
- finalBag.add(intermediateTuple);
- }
-
- Double result = finalVar.exec(TupleFactory.getInstance().newTuple(finalBag));
-
- Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
- }
-
- // make sure intermediate works, where initial just passes through a single tuple, and intermediate receives a large bag of the resulting tuples
- @Test
- public void varIntAlgebraicIntermediateTest() throws Exception {
- IntVAR.Initial initialVar = new IntVAR.Initial();
- IntVAR.Intermediate intermediateVar = new IntVAR.Intermediate();
- IntVAR.Final finalVar = new IntVAR.Final();
-
-
- DataBag intermediateBag = BagFactory.getInstance().newDefaultBag();
-
- for (int i=1; i<=1000; i++)
- {
- DataBag bag;
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, (int)i);
- bag = BagFactory.getInstance().newDefaultBag();
- bag.add(t);
- Tuple input = TupleFactory.getInstance().newTuple(bag);
- Tuple intermediateTuple = initialVar.exec(input);
- intermediateBag.add(intermediateTuple);
- }
-
- Tuple intermediateTuple = intermediateVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
- intermediateBag = BagFactory.getInstance().newDefaultBag(Arrays.asList(intermediateTuple));
- Double result = finalVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
-
- Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
- }
-
- // make sure final works, where initial just passes through a single tuple, intermediate does the same, and final receives the remainder
- @Test
- public void varIntAlgebraicFinalTest() throws Exception {
- IntVAR.Initial initialVar = new IntVAR.Initial();
- IntVAR.Intermediate intermediateVar = new IntVAR.Intermediate();
- IntVAR.Final finalVar = new IntVAR.Final();
-
- DataBag finalBag = BagFactory.getInstance().newDefaultBag();
-
- for (int i=1; i<=1000; i++)
- {
- DataBag bag;
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, (int)i);
- bag = BagFactory.getInstance().newDefaultBag();
- bag.add(t);
- Tuple input = TupleFactory.getInstance().newTuple(bag);
- Tuple intermediateTuple = initialVar.exec(input);
- DataBag intermediateBag = BagFactory.getInstance().newDefaultBag();
- intermediateBag.add(intermediateTuple);
- intermediateTuple = intermediateVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
- finalBag.add(intermediateTuple);
- }
-
- Double result = finalVar.exec(TupleFactory.getInstance().newTuple(finalBag));
-
- Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
- }
-
- // make sure intermediate works, where initial just passes through a single tuple, and intermediate receives a large bag of the resulting tuples
- @Test
- public void varLongAlgebraicIntermediateTest() throws Exception {
- LongVAR.Initial initialVar = new LongVAR.Initial();
- LongVAR.Intermediate intermediateVar = new LongVAR.Intermediate();
- LongVAR.Final finalVar = new LongVAR.Final();
-
-
- DataBag intermediateBag = BagFactory.getInstance().newDefaultBag();
-
- for (int i=1; i<=1000; i++)
- {
- DataBag bag;
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, (long)i);
- bag = BagFactory.getInstance().newDefaultBag();
- bag.add(t);
- Tuple input = TupleFactory.getInstance().newTuple(bag);
- Tuple intermediateTuple = initialVar.exec(input);
- intermediateBag.add(intermediateTuple);
- }
-
- Tuple intermediateTuple = intermediateVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
- intermediateBag = BagFactory.getInstance().newDefaultBag(Arrays.asList(intermediateTuple));
- Double result = finalVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
-
- Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
- }
-
- // make sure final works, where initial just passes through a single tuple, intermediate does the same, and final receives the remainder
- @Test
- public void varLongAlgebraicFinalTest() throws Exception {
- LongVAR.Initial initialVar = new LongVAR.Initial();
- LongVAR.Intermediate intermediateVar = new LongVAR.Intermediate();
- LongVAR.Final finalVar = new LongVAR.Final();
-
- DataBag finalBag = BagFactory.getInstance().newDefaultBag();
-
- for (int i=1; i<=1000; i++)
- {
- DataBag bag;
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, (long)i);
- bag = BagFactory.getInstance().newDefaultBag();
- bag.add(t);
- Tuple input = TupleFactory.getInstance().newTuple(bag);
- Tuple intermediateTuple = initialVar.exec(input);
- DataBag intermediateBag = BagFactory.getInstance().newDefaultBag();
- intermediateBag.add(intermediateTuple);
- intermediateTuple = intermediateVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
- finalBag.add(intermediateTuple);
- }
-
- Double result = finalVar.exec(TupleFactory.getInstance().newTuple(finalBag));
-
- Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
- }
- }
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/stats/WilsonBinConfTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/stats/WilsonBinConfTests.java b/test/pig/datafu/test/pig/stats/WilsonBinConfTests.java
deleted file mode 100644
index 34e0a48..0000000
--- a/test/pig/datafu/test/pig/stats/WilsonBinConfTests.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.stats;
-
-import static org.testng.Assert.*;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.test.pig.PigTests;
-
-public class WilsonBinConfTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- define WilsonBinConf datafu.pig.stats.WilsonBinConf('$alpha');
-
- data = load 'input' as (successes:long, totals:long);
- --describe data;
-
- data_out = FOREACH data GENERATE WilsonBinConf(successes, totals) as interval;
- data_out = FOREACH data_out GENERATE FLATTEN(interval);
-
- store data_out into 'output';
- */
- @Multiline private String wilsonBinConf;
-
- @Test
- public void wilsonTest() throws Exception
- {
- PigTest test = createPigTestFromString(wilsonBinConf,
- "alpha=0.05"); // alpha is 0.05 for 95% confidence
-
- writeLinesToFile("input",
- "1\t1",
- "1\t2",
- "50\t100",
- "500\t1000",
- "999\t1000",
- "1000\t1000",
- "998\t1000");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * library(Hmisc)
- *
- * binconf(50,100)
- * binconf(500,1000)
- *
- */
- List<String> expectedOutput = new ArrayList<String>();
- expectedOutput.add("0.05129,1.00000");
- expectedOutput.add("0.02565,0.97435");
- expectedOutput.add("0.40383,0.59617");
- expectedOutput.add("0.46907,0.53093");
- expectedOutput.add("0.99436,0.99995");
- expectedOutput.add("0.99617,1.00000");
- expectedOutput.add("0.99274,0.99945");
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- Iterator<String> expectationIterator = expectedOutput.iterator();
- for (Tuple t : output)
- {
- assertTrue(expectationIterator.hasNext());
- Double lower = (Double)t.get(0);
- Double upper = (Double)t.get(1);
- assertEquals(String.format("%.5f,%.5f",lower,upper),expectationIterator.next());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/stats/entropy/AbstractEntropyTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/stats/entropy/AbstractEntropyTests.java b/test/pig/datafu/test/pig/stats/entropy/AbstractEntropyTests.java
deleted file mode 100644
index 6e512d4..0000000
--- a/test/pig/datafu/test/pig/stats/entropy/AbstractEntropyTests.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.stats.entropy;
-
-import static org.testng.Assert.*;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.pig.data.Tuple;
-
-import org.apache.pig.backend.executionengine.ExecException;
-
-import datafu.test.pig.PigTests;
-
-public abstract class AbstractEntropyTests extends PigTests
-{
- protected void verifyEqualEntropyOutput(List<Double> expectedOutput, List<Tuple> output, int digits) throws ExecException {
- assertEquals(expectedOutput.size(), output.size());
- Iterator<Double> expectationIterator = expectedOutput.iterator();
- String formatDigits = "%." + digits + "f";
- for (Tuple t : output)
- {
- Double entropy = (Double)t.get(0);
- assertEquals(String.format(formatDigits,entropy),String.format(formatDigits, expectationIterator.next()));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/stats/entropy/ChaoShenEntropyTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/stats/entropy/ChaoShenEntropyTests.java b/test/pig/datafu/test/pig/stats/entropy/ChaoShenEntropyTests.java
deleted file mode 100644
index 0030bf2..0000000
--- a/test/pig/datafu/test/pig/stats/entropy/ChaoShenEntropyTests.java
+++ /dev/null
@@ -1,373 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.stats.entropy;
-
-import static org.testng.Assert.*;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.test.pig.PigTests;
-
-/*
- * R's entropy library: http://cran.r-project.org/web/packages/entropy/entropy.pdf
- * used as our test benchmark
- */
-public class ChaoShenEntropyTests extends AbstractEntropyTests
-{
- /**
- register $JAR_PATH
-
- define Entropy datafu.pig.stats.entropy.Entropy('$type','$base');
-
- data = load 'input' as (val:double);
- --describe data;
- data_grouped = GROUP data ALL;
- data_out = FOREACH data_grouped {
- data_ordered = ORDER data BY val;
- GENERATE Entropy(data_ordered);
- };
- store data_out into 'output';
- */
- @Multiline private String entropy;
-
- @Test
- public void uniqValChaoShenEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(entropy, "type=chaosh", "base=log");
-
- writeLinesToFile("input",
- "98.94791",
- "38.61010",
- "97.10575",
- "62.28313",
- "38.83960",
- "32.05370",
- "96.10962",
- "28.72388",
- "96.65888",
- "20.41135");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > v=c(98.94791,38.61010,97.10575,62.28313,38.83960,32.05370,96.10962,28.72388,96.65888,20.41135)
- * > table(v)
- * v
- * 20.41135 28.72388 32.0537 38.6101 38.8396 62.28313 96.10962 96.65888 97.10575 98.94791
- * 1 1 1 1 1 1 1 1 1 1
- * > count=c(1,1,1,1,1,1,1,1,1,1)
- * > library(entropy)
- * > entropy(count,count/sum(count),c("CS"),c("log"))
- * [1] 4.816221
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(4.816221);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void singleValChaoShenEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(entropy, "type=chaosh", "base=log");
-
- writeLinesToFile("input",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > v=c(98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791)
- * > table(v)
- * v
- * 98.94791
- * 10
- * > count=(10)
- * > entropy(count,count/sum(count),c("CS"),c("log"))
- * [1] 0
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(0.0);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void dupValChaoShenEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(entropy,"type=chaosh", "base=log");
-
- writeLinesToFile("input",
- "98.94791",
- "38.61010",
- "97.10575",
- "62.28313",
- "38.61010",
- "32.05370",
- "96.10962",
- "38.61010",
- "96.10962",
- "20.41135");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > v=c(98.94791,38.61010,97.10575,62.28313,38.61010,32.05370,96.10962,38.61010,96.10962,20.41135)
- * > table(v)
- * v
- * 20.41135 28.72388 32.0537 38.6101 38.8396 62.28313 96.10962 96.65888 97.10575 98.94791
- * 1 1 3 1 2 1 1
- * > count=c(1,1,3,1,2,1,1)
- * > library(entropy)
- * > entropy(count,count/sum(count),c("CS"),c("log"))
- * [1] 2.57429
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(2.57429);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
-
- @Test
- public void emptyInputBagChaoShenEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(entropy, "type=chaosh", "base=log");
-
- writeLinesToFile("input"
- );
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > v=c()
- * > table(v)
- * < table of extent 0 >
- * > count=c()
- * > library(entropy)
- * > entropy(count,count/sum(count),c("CS"),c("log"))
- * [1] 0
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void singleElemInputBagChaoShenEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(entropy, "type=chaosh", "base=log");
-
- writeLinesToFile("input",
- "98.94791");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > count=c(1)
- * > library(entropy)
- * > entropy(count,count/sum(count),c("CS"),c("log"))
- * [1] 0
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(0.0);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- /**
- register $JAR_PATH
-
- define Entropy datafu.pig.stats.entropy.Entropy('$type','$base');
-
- data = load 'input' as (x:chararray, y:double);
- --describe data;
- data_grouped = GROUP data ALL;
- data_out = FOREACH data_grouped {
- data_ordered = ORDER data BY x,y;
- GENERATE Entropy(data_ordered);
- };
- store data_out into 'output';
- */
- @Multiline private String pairEntropy;
-
- @Test
- public void dupPairValChaoShenEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(pairEntropy, "type=chaosh", "base=log");
-
- writeLinesToFile("input",
- "hadoop 98.94791",
- "bigdata 38.61010",
- "hadoop 97.10575",
- "datafu 32.05370",
- "bigdata 38.61010",
- "datafu 32.05370",
- "datafu 32.05370",
- "hadoop 38.61010",
- "pig 96.10962",
- "pig 20.41135");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- * > t <- data.table(x=c("hadoop","bigdata","hadoop","datafu","bigdata","datafu","datafu","hadoop","pig","pig"),y=c(98.94791,38.61010,97.10575,32.05370,38.61010,32.05370,32.05370,38.61010,96.10962,20.41135))
- * > t <- t[order(x,y)]
- * > count<-c(2,3,1,1,1,1,1)
- * > library(entropy)
- * > entropy(count,count/sum(count),c("CS"),c("log"))
- * [1] 2.57429
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(2.57429);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void dupValChaoShenEntropoyLog2Test() throws Exception
- {
- PigTest test = createPigTestFromString(entropy,"type=chaosh", "base=log2");
-
- writeLinesToFile("input",
- "98.94791",
- "38.61010",
- "97.10575",
- "62.28313",
- "38.61010",
- "32.05370",
- "96.10962",
- "38.61010",
- "96.10962",
- "20.41135");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > v=c(98.94791,38.61010,97.10575,62.28313,38.61010,32.05370,96.10962,38.61010,96.10962,20.41135)
- * > table(v)
- * v
- * 20.41135 28.72388 32.0537 38.6101 38.8396 62.28313 96.10962 96.65888 97.10575 98.94791
- * 1 1 3 1 2 1 1
- * > count=c(1,1,3,1,2,1,1)
- * > freqs=count/sum(count)
- * > library(entropy)
- * > entropy(count,count/sum(count),c("CS"),c("log2"))
- * [1] 3.713915
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(3.713915);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void dupValChaoShenEntropoyLog10Test() throws Exception
- {
- PigTest test = createPigTestFromString(entropy, "type=chaosh", "base=log10");
-
- writeLinesToFile("input",
- "98.94791",
- "38.61010",
- "97.10575",
- "62.28313",
- "38.61010",
- "32.05370",
- "96.10962",
- "38.61010",
- "96.10962",
- "20.41135");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > v=c(98.94791,38.61010,97.10575,62.28313,38.61010,32.05370,96.10962,38.61010,96.10962,20.41135)
- * > table(v)
- * v
- * 20.41135 28.72388 32.0537 38.6101 38.8396 62.28313 96.10962 96.65888 97.10575 98.94791
- * 1 1 3 1 2 1 1
- * > count=c(1,1,3,1,2,1,1)
- * > library(entropy)
- * > entropy(count,count/sum(count),c("CS"),c("log10"))
- * [1] 1.118
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(1.118);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/stats/entropy/CondEntropyTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/stats/entropy/CondEntropyTests.java b/test/pig/datafu/test/pig/stats/entropy/CondEntropyTests.java
deleted file mode 100644
index c6d62df..0000000
--- a/test/pig/datafu/test/pig/stats/entropy/CondEntropyTests.java
+++ /dev/null
@@ -1,412 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.stats.entropy;
-
-import static org.testng.Assert.*;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.test.pig.PigTests;
-
-
-/*
- * Use R function to compute condition entropy as the test benchmark
- * http://cran.r-project.org/web/packages/infotheo/infotheo.pdf
- */
-public class CondEntropyTests extends AbstractEntropyTests
-{
- /**
- register $JAR_PATH
-
- define CondEntropy datafu.pig.stats.entropy.CondEntropy();
-
- data = load 'input' as (valX:double, valY:chararray);
- --describe data;
- data_grouped = GROUP data ALL;
- data_out = FOREACH data_grouped {
- data_ordered = ORDER data BY *;
- GENERATE CondEntropy(data_ordered);
- };
- store data_out into 'output';
- */
- @Multiline private String condEntropy;
-
- @Test
- public void uniqValEmpiricalCondEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(condEntropy);
-
- writeLinesToFile("input",
- "98.94791 click",
- "38.61010 view",
- "97.10575 view",
- "62.28313 click",
- "38.83960 click",
- "32.05370 view",
- "96.10962 view",
- "28.72388 click",
- "96.65888 view",
- "20.41135 click");
-
- test.runScript();
-
- /*
- * library(infotheo)
- * X=c("98.94791","38.61010","97.10575","62.28313","38.83960","32.05370","96.10962","28.72388","96.65888","20.41135")
- * Y=c("click","view","view","click","click","view","view","click","view","click")
- * condentropy(Y,X)
- * [1] 0
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(0.0);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void singleValEmpiricalCondEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(condEntropy);
-
- writeLinesToFile("input",
- "98.94791 click",
- "98.94791 click",
- "98.94791 click",
- "98.94791 click",
- "98.94791 click",
- "98.94791 click",
- "98.94791 click",
- "98.94791 click",
- "98.94791 click",
- "98.94791 click");
-
- test.runScript();
-
- /*
- * library(infotheo)
- * X=c("98.94791","98.94791","98.94791","98.94791","98.94791","98.94791","98.94791","98.94791","98.94791","98.94791")
- * Y=c("click","click","click","click","click","click","click","click","click","click")
- * condentropy(Y,X)
- * [1] 0
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(0.0);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void dupValEmpiricalCondEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(condEntropy);
-
- writeLinesToFile("input",
- "98.94791 click",
- "38.61010 click",
- "97.10575 view",
- "62.28313 view",
- "38.61010 view",
- "32.05370 view",
- "96.10962 click",
- "38.61010 click",
- "96.10962 view",
- "20.41135 click");
-
- test.runScript();
-
- /*
- * library(infotheo)
- * X=c("98.94791","38.61010","97.10575","62.28313","38.61010","32.05370","96.10962","38.61010","96.10962","20.41135")
- * Y=c("click","click","view","view","view","view","click","click","view","click")
- * condentropy(Y,X)
- * [1] 0.3295837
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(0.3295837);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void emptyInputBagEmpiricalCondEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(condEntropy);
-
- writeLinesToFile("input"
- );
-
- test.runScript();
-
- List<Double> expectedOutput = new ArrayList<Double>();
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void singleElemInputBagEmpiricalCondEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(condEntropy);
-
- writeLinesToFile("input",
- "98.94791 view");
-
- test.runScript();
- /*
- * library(infotheo)
- * X = c("98.94791")
- * Y = c("view")
- * condentropy(Y,X)
- * [1] 0
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(0.0);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- /**
- register $JAR_PATH
-
- define Entropy datafu.pig.stats.entropy.CondEntropy();
-
- data = load 'input' as (valX1:chararray, valX2:chararray, valY:chararray);
- data = foreach data generate (valX1, valX2) as X, valY as Y;
- --describe data;
- data_grouped = GROUP data ALL;
- data_out = FOREACH data_grouped {
- data_ordered = ORDER data BY *;
- GENERATE Entropy(data_ordered);
- };
-
- store data_out into 'output';
- */
- @Multiline private String pairCondEntropy;
-
- @Test
- public void dupPairValEmpiricalCondEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(pairCondEntropy);
-
- writeLinesToFile("input",
- "hadoop bigdata click",
- "hadoop pig view",
- "hadoop datafu click",
- "datafu pig click",
- "bigdata pig view",
- "datafu pig click",
- "datafu pig view",
- "hadoop bigdata view",
- "pig datafu view",
- "pig datafu view");
-
- test.runScript();
-
- /*
- * library(infotheo)
- * X=c("hadoop bigdata","hadoop pig","hadoop datafu","datafu pig","bigdata pig","datafu pig","datafu pig","hadoop bigdata","pig datafu","pig datafu")
- * Y=c("click","view","click","click","view","click","view","view","view","view")
- * condentropy(X,Y)
- * [1] 0.3295837
- */
-
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(0.3295837);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- /**
- register $JAR_PATH
-
- define CondEntropy datafu.pig.stats.entropy.CondEntropy('$type','$base');
-
- data = load 'input' as (valX:double, valY:chararray);
- --describe data;
- data_grouped = GROUP data ALL;
- data_out = FOREACH data_grouped {
- data_ordered = ORDER data BY *;
- GENERATE CondEntropy(data_ordered);
- };
- store data_out into 'output';
- */
- @Multiline private String condLogEntropy;
-
- @Test
- public void dupValEmpiricalCondEntropoyLog2Test() throws Exception
- {
- PigTest test = createPigTestFromString(condLogEntropy, "type=empirical", "base=log2");
-
- writeLinesToFile("input",
- "98.94791 click",
- "38.61010 click",
- "97.10575 view",
- "62.28313 view",
- "38.61010 view",
- "32.05370 view",
- "96.10962 click",
- "38.61010 click",
- "96.10962 view",
- "20.41135 click");
-
- test.runScript();
-
- /*
- * library(infotheo)
- * X=c("98.94791","38.61010","97.10575","62.28313","38.61010","32.05370","96.10962","38.61010","96.10962","20.41135")
- * Y=c("click","click","view","view","view","view","click","click","view","click")
- * condentropy(Y,X)/log(2)
- * [1] 0.4754888
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(0.4754888);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void dupValEmpiricalCondEntropoyLog10Test() throws Exception
- {
- PigTest test = createPigTestFromString(condLogEntropy, "type=empirical", "base=log10");
-
- writeLinesToFile("input",
- "98.94791 click",
- "38.61010 click",
- "97.10575 view",
- "62.28313 view",
- "38.61010 view",
- "32.05370 view",
- "96.10962 click",
- "38.61010 click",
- "96.10962 view",
- "20.41135 click");
-
- test.runScript();
-
- /*
- * library(infotheo)
- * X=c("98.94791","38.61010","97.10575","62.28313","38.61010","32.05370","96.10962","38.61010","96.10962","20.41135")
- * Y=c("click","click","view","view","view","view","click","click","view","click")
- * condentropy(Y,X)/log(10)
- * [1] 0.1431364
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(0.1431364);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- /**
- register $JAR_PATH
-
- define CondEntropy datafu.pig.stats.entropy.CondEntropy();
-
- data = load 'input' as (valX:double, valY:chararray);
- --describe data;
- data_grouped = GROUP data ALL;
- data_out = FOREACH data_grouped {
- GENERATE CondEntropy(data);
- };
- store data_out into 'output';
- */
- @Multiline private String noOrderCondEntropy;
-
- @Test
- public void noOrderEmpiricalCondEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(noOrderCondEntropy);
-
- writeLinesToFile("input",
- "98.94791 click",
- "38.61010 view",
- "97.10575 view",
- "62.28313 click",
- "38.83960 click",
- "32.05370 view",
- "96.10962 view",
- "28.72388 click",
- "96.65888 view",
- "20.41135 click");
-
- try {
- test.runScript();
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- fail( "Testcase should fail");
- } catch(Exception ex) {}
- }
-
- /**
- register $JAR_PATH
-
- define CondEntropy datafu.pig.stats.entropy.CondEntropy();
-
- data = load 'input' as (valX:double);
- --describe data;
- data_grouped = GROUP data ALL;
- data_out = FOREACH data_grouped {
- data_ordered = ORDER data BY *;
- GENERATE CondEntropy(data_ordered);
- };
- store data_out into 'output';
- */
- @Multiline private String invalidInputCondEntropy;
-
- @Test
- public void invalidInputEmpiricalCondEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(invalidInputCondEntropy);
-
- writeLinesToFile("input",
- "98.94791",
- "38.61010",
- "97.10575",
- "62.28313",
- "38.83960",
- "32.05370",
- "96.10962",
- "28.72388",
- "96.65888",
- "20.41135");
-
- try {
- test.runScript();
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- fail( "Testcase should fail");
- } catch(Exception ex) {
- assertTrue(ex.getMessage().indexOf("The field schema of the input tuple is null or its size is not 2") >= 0);
- }
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/stats/entropy/EmpiricalCountEntropyTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/stats/entropy/EmpiricalCountEntropyTests.java b/test/pig/datafu/test/pig/stats/entropy/EmpiricalCountEntropyTests.java
deleted file mode 100644
index 52fdcce..0000000
--- a/test/pig/datafu/test/pig/stats/entropy/EmpiricalCountEntropyTests.java
+++ /dev/null
@@ -1,585 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.stats.entropy;
-
-import static org.testng.Assert.*;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.test.pig.PigTests;
-
-/*
- * R's entropy library: http://cran.r-project.org/web/packages/entropy/entropy.pdf
- * used as our test benchmark
- */
-public class EmpiricalCountEntropyTests extends AbstractEntropyTests
-{
- /**
- register $JAR_PATH
-
- define Entropy datafu.pig.stats.entropy.EmpiricalCountEntropy();
-
- data = load 'input' as (val:double);
- --describe data;
- data_grouped = GROUP data BY val;
- data_cnt = FOREACH data_grouped GENERATE COUNT(data) AS cnt;
- data_cnt_grouped = GROUP data_cnt ALL;
- data_out = FOREACH data_cnt_grouped GENERATE Entropy(data_cnt);
- store data_out into 'output';
- */
- @Multiline private String entropy;
-
- @Test
- public void uniqValEntropyTest() throws Exception
- {
- PigTest test = createPigTestFromString(entropy);
-
- writeLinesToFile("input",
- "98.94791",
- "38.61010",
- "97.10575",
- "62.28313",
- "38.83960",
- "32.05370",
- "96.10962",
- "28.72388",
- "96.65888",
- "20.41135");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > v=c(98.94791,38.61010,97.10575,62.28313,38.83960,32.05370,96.10962,28.72388,96.65888,20.41135)
- * > table(v)
- * v
- * 20.41135 28.72388 32.0537 38.6101 38.8396 62.28313 96.10962 96.65888 97.10575 98.94791
- * 1 1 1 1 1 1 1 1 1 1
- * > count=c(1,1,1,1,1,1,1,1,1,1)
- * > library(entropy)
- * > entropy(count)
- * [1] 2.302585
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(2.302585);
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void singleValEntropyTest() throws Exception
- {
- PigTest test = createPigTestFromString(entropy);
-
- writeLinesToFile("input",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > v=c(98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791)
- * > table(v)
- * v
- * 98.94791
- * 10
- * > count=(10)
- * > library(entropy)
- * > entropy(count)
- * [1] 0
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(0.0);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void dupValEntropyTest() throws Exception
- {
- PigTest test = createPigTestFromString(entropy);
-
- writeLinesToFile("input",
- "98.94791",
- "38.61010",
- "97.10575",
- "62.28313",
- "38.61010",
- "32.05370",
- "96.10962",
- "38.61010",
- "96.10962",
- "20.41135");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > v=c(98.94791,38.61010,97.10575,62.28313,38.61010,32.05370,96.10962,38.61010,96.10962,20.41135)
- * > table(v)
- * v
- * 20.41135 32.0537 38.6101 62.28313 96.10962 97.10575 98.94791
- * 1 1 3 1 2 1 1
- * > count=c(1,1,3,1,2,1,1)
- * > library(entropy)
- * > entropy(count)
- * [1] 1.834372
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(1.834372);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void emptyInputBagEntropyTest() throws Exception
- {
- PigTest test = createPigTestFromString(entropy);
-
- writeLinesToFile("input"
- );
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > v=c()
- * > table(v)
- * < table of extent 0 >
- * > count=c()
- * > library(entropy)
- * > entropy(count)
- * [1] 0
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void singleElemInputBagEntropyTest() throws Exception
- {
- PigTest test = createPigTestFromString(entropy);
-
- writeLinesToFile("input",
- "98.94791");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > count=c(1)
- * > library(entropy)
- * > entropy(count)
- * [1] 0
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(0.0);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- /**
- register $JAR_PATH
-
- define Entropy datafu.pig.stats.entropy.EmpiricalCountEntropy('$base');
-
- data = load 'input' as (x:chararray, y:double);
- --describe data;
- data_grouped = GROUP data BY (x, y);
- data_cnt = FOREACH data_grouped GENERATE COUNT(data);
- data_cnt_grouped = GROUP data_cnt ALL;
- data_out = FOREACH data_cnt_grouped GENERATE Entropy(data_cnt);
- store data_out into 'output';
- */
- @Multiline private String pairLogEntropy;
-
- @Test
- public void dupPairValEntropyTest() throws Exception
- {
- PigTest test = createPigTestFromString(pairLogEntropy, "base=log");
-
- writeLinesToFile("input",
- "hadoop 98.94791",
- "bigdata 38.61010",
- "hadoop 97.10575",
- "datafu 32.05370",
- "bigdata 38.61010",
- "datafu 32.05370",
- "datafu 32.05370",
- "hadoop 38.61010",
- "pig 96.10962",
- "pig 20.41135");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- * > t <- data.table(x=c("hadoop","bigdata","hadoop","datafu","bigdata","datafu","datafu","hadoop","pig","pig"),y=c(98.94791,38.61010,97.10575,32.05370,38.61010,32.05370,32.05370,38.61010,96.10962,20.41135))
- * > t <- t[order(x,y)]
- * > count<-c(2,3,1,1,1,1,1)
- * > library(entropy)
- * > entropy(count)
- * [1] 1.834372
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(1.834372);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- /**
- register $JAR_PATH
-
- define Entropy datafu.pig.stats.entropy.EmpiricalCountEntropy('$base');
-
- data = load 'input' as (val:double);
- --describe data;
- data_grouped = GROUP data BY val;
- data_cnt = FOREACH data_grouped GENERATE COUNT(data) AS cnt;
- data_cnt_grouped = GROUP data_cnt ALL;
- data_out = FOREACH data_cnt_grouped GENERATE Entropy(data_cnt);
- store data_out into 'output';
- */
- @Multiline private String logEntropy;
-
- @Test
- public void dupValEntropyLog2Test() throws Exception
- {
- PigTest test = createPigTestFromString(logEntropy, "base=log2");
-
- writeLinesToFile("input",
- "98.94791",
- "38.61010",
- "97.10575",
- "62.28313",
- "38.61010",
- "32.05370",
- "96.10962",
- "38.61010",
- "96.10962",
- "20.41135");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > v=c(98.94791,38.61010,97.10575,62.28313,38.61010,32.05370,96.10962,38.61010,96.10962,20.41135)
- * > table(v)
- * v
- * 20.41135 32.0537 38.6101 62.28313 96.10962 97.10575 98.94791
- * 1 1 3 1 2 1 1
- * > count=c(1,1,3,1,2,1,1)
- * > library(entropy)
- * > entropy(count, count/sum(count), c("ML"),c("log2"))
- * [1] 2.646439
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(2.646439);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void dupValEntropyLog10Test() throws Exception
- {
- PigTest test = createPigTestFromString(logEntropy, "base=log10");
-
- writeLinesToFile("input",
- "98.94791",
- "38.61010",
- "97.10575",
- "62.28313",
- "38.61010",
- "32.05370",
- "96.10962",
- "38.61010",
- "96.10962",
- "20.41135");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > v=c(98.94791,38.61010,97.10575,62.28313,38.61010,32.05370,96.10962,38.61010,96.10962,20.41135)
- * > table(v)
- * v
- * 20.41135 32.0537 38.6101 62.28313 96.10962 97.10575 98.94791
- * 1 1 3 1 2 1 1
- * > count=c(1,1,3,1,2,1,1)
- * > library(entropy)
- * > entropy(count, count/sum(count), c("ML"),c("log10"))
- * [1] 0.7966576
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(0.7966576);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
-
- /**
- register $JAR_PATH
-
- define Entropy datafu.pig.stats.entropy.EmpiricalCountEntropy();
-
- data_cnt = load 'input' as (val:int);
- --describe data_cnt;
- data_cnt_grouped = GROUP data_cnt ALL;
- data_out = FOREACH data_cnt_grouped GENERATE Entropy(data_cnt);
- store data_out into 'output';
- */
- @Multiline private String rawValidInputEntropy;
-
- @Test
- public void rawValidInputEntropyTest() throws Exception
- {
- PigTest test = createPigTestFromString(rawValidInputEntropy);
-
- writeLinesToFile("input",
- "0",
- "38",
- "0",
- "62",
- "38",
- "32",
- "96",
- "38",
- "96",
- "0");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > count=c(0, 38, 0, 62, 38, 32, 96, 38, 96, 0)
- * > library(entropy)
- * > entropy(count)
- * [1] 1.846901
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(1.846901);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- /**
- register $JAR_PATH
-
- define Entropy datafu.pig.stats.entropy.EmpiricalCountEntropy();
-
- data_cnt = load 'input' as (val:double);
- --describe data_cnt;
- data_cnt_grouped = GROUP data_cnt ALL;
- data_out = FOREACH data_cnt_grouped GENERATE Entropy(data_cnt);
- store data_out into 'output';
- */
- @Multiline private String rawInvalidTypeInputEntropy;
-
- @Test
- public void rawInvalidTypeInputEntropyTest() throws Exception
- {
- PigTest test = createPigTestFromString(rawInvalidTypeInputEntropy);
-
- writeLinesToFile("input",
- "0.0",
- "38.0",
- "0.0",
- "62.0",
- "38.0",
- "32.001",
- "96.002",
- "38.01",
- "96.00001",
- "0.0");
- try {
- test.runScript();
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- fail( "Testcase should fail");
- } catch (Exception ex) {
- assertTrue(ex.getMessage().indexOf("Expect the type of the input tuple to be of ([int, long]), but instead found double") >= 0);
- }
- }
-
- @Test
- public void rawInValidInputValueEntropyTest() throws Exception
- {
- PigTest test = createPigTestFromString(rawValidInputEntropy);
-
- writeLinesToFile("input",
- "0",
- "-38",
- "0",
- "62",
- "38",
- "32",
- "96",
- "38",
- "96",
- "0");
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > count=c(0, -38, 0, 62, 38, 32, 96, 38, 96, 0)
- * > library(entropy)
- * > entropy(ifelse(count>0,count,0))
- * [1] 1.693862
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(1.693862);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
-
- }
-
- /**
- register $JAR_PATH
-
- define Entropy datafu.pig.stats.entropy.EmpiricalCountEntropy();
-
- data_cnt = load 'input' as (f1:chararray, f2:chararray);
- --describe data_cnt;
- data_cnt_grouped = GROUP data_cnt ALL;
- data_out = FOREACH data_cnt_grouped GENERATE Entropy(data_cnt);
- store data_out into 'output';
- */
- @Multiline private String invalidInputSchemaEntropy;
-
- @Test
- public void invalidInputSchemaEntropyTest() throws Exception
- {
- PigTest test = createPigTestFromString(invalidInputSchemaEntropy);
-
- writeLinesToFile("input",
- "hadoop 98.94791",
- "bigdata 38.61010",
- "hadoop 97.10575",
- "datafu 32.05370",
- "bigdata 38.61010",
- "datafu 32.05370",
- "datafu 32.05370");
-
- try {
- test.runScript();
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- fail( "Testcase should fail");
- } catch (Exception ex) {
- assertTrue(ex.getMessage().indexOf("The field schema of the input tuple is null or its size is not 1") >= 0);
- }
- }
-
- /**
- register $JAR_PATH
-
- define Entropy datafu.pig.stats.entropy.EmpiricalCountEntropy();
-
- data = load 'input' as (val:double);
- --describe data;
- data_grouped = GROUP data BY val;
- data_cnt = FOREACH data_grouped GENERATE COUNT(data) AS cnt;
- data_cnt_grouped = GROUP data_cnt ALL;
- data_out = FOREACH data_cnt_grouped {
- data_cnt_ordered = order data_cnt by *;
- GENERATE Entropy(data_cnt_ordered);
- }
- store data_out into 'output';
- */
- @Multiline private String accumulatedEntropy;
-
- @Test
- public void accumulatedEntropyTest() throws Exception
- {
- PigTest test = createPigTestFromString(accumulatedEntropy);
-
- writeLinesToFile("input",
- "98.94791",
- "38.61010",
- "97.10575",
- "62.28313",
- "38.61010",
- "32.05370",
- "96.10962",
- "38.61010",
- "96.10962",
- "20.41135");
-
- test.runScript();
-
- List<Double> expectedOutput = new ArrayList<Double>();
- //the same output as @test dupValEntropyTest
- expectedOutput.add(1.834372);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/stats/entropy/EntropyTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/stats/entropy/EntropyTests.java b/test/pig/datafu/test/pig/stats/entropy/EntropyTests.java
deleted file mode 100644
index e1611a7..0000000
--- a/test/pig/datafu/test/pig/stats/entropy/EntropyTests.java
+++ /dev/null
@@ -1,429 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.stats.entropy;
-
-import static org.testng.Assert.*;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.test.pig.PigTests;
-
-/*
- * R's entropy library: http://cran.r-project.org/web/packages/entropy/entropy.pdf
- * used as our test benchmark
- */
-public class EntropyTests extends AbstractEntropyTests
-{
- /**
- register $JAR_PATH
-
- define Entropy datafu.pig.stats.entropy.Entropy();
-
- data = load 'input' as (val:double);
- --describe data;
- data_grouped = GROUP data ALL;
- data_out = FOREACH data_grouped {
- data_ordered = ORDER data BY val;
- GENERATE Entropy(data_ordered);
- };
- store data_out into 'output';
- */
- @Multiline private String entropy;
-
- @Test
- public void uniqValEmpiricalEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(entropy);
-
- writeLinesToFile("input",
- "98.94791",
- "38.61010",
- "97.10575",
- "62.28313",
- "38.83960",
- "32.05370",
- "96.10962",
- "28.72388",
- "96.65888",
- "20.41135");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > v=c(98.94791,38.61010,97.10575,62.28313,38.83960,32.05370,96.10962,28.72388,96.65888,20.41135)
- * > table(v)
- * v
- * 20.41135 28.72388 32.0537 38.6101 38.8396 62.28313 96.10962 96.65888 97.10575 98.94791
- * 1 1 1 1 1 1 1 1 1 1
- * > count=c(1,1,1,1,1,1,1,1,1,1)
- * > library(entropy)
- * > entropy(count)
- * [1] 2.302585
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(2.302585);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void singleValEmpiricalEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(entropy);
-
- writeLinesToFile("input",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791",
- "98.94791");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > v=c(98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791)
- * > table(v)
- * v
- * 98.94791
- * 10
- * > count=(10)
- * > library(entropy)
- * > entropy(count)
- * [1] 0
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(0.0);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void dupValEmpiricalEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(entropy);
-
- writeLinesToFile("input",
- "98.94791",
- "38.61010",
- "97.10575",
- "62.28313",
- "38.61010",
- "32.05370",
- "96.10962",
- "38.61010",
- "96.10962",
- "20.41135");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > v=c(98.94791,38.61010,97.10575,62.28313,38.61010,32.05370,96.10962,38.61010,96.10962,20.41135)
- * > table(v)
- * v
- * 20.41135 32.0537 38.6101 62.28313 96.10962 97.10575 98.94791
- * 1 1 3 1 2 1 1
- * > count=c(1,1,3,1,2,1,1)
- * > library(entropy)
- * > entropy(count)
- * [1] 1.834372
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(1.834372);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void emptyInputBagEmpiricalEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(entropy);
-
- writeLinesToFile("input"
- );
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > v=c()
- * > table(v)
- * < table of extent 0 >
- * > count=c()
- * > library(entropy)
- * > entropy(count)
- * [1] 0
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void singleElemInputBagEmpiricalEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(entropy);
-
- writeLinesToFile("input",
- "98.94791");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > count=c(1)
- * > library(entropy)
- * > entropy(count)
- * [1] 0
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(0.0);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- /**
- register $JAR_PATH
-
- define Entropy datafu.pig.stats.entropy.Entropy('$type', '$base');
-
- data = load 'input' as (x:chararray, y:double);
- --describe data;
- data_grouped = GROUP data ALL;
- data_out = FOREACH data_grouped {
- data_ordered = ORDER data BY x,y;
- GENERATE Entropy(data_ordered);
- };
- store data_out into 'output';
- */
- @Multiline private String pairEntropy;
-
- @Test
- public void dupPairValEmpiricalEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(pairEntropy, "type=empirical", "base=log");
-
- writeLinesToFile("input",
- "hadoop 98.94791",
- "bigdata 38.61010",
- "hadoop 97.10575",
- "datafu 32.05370",
- "bigdata 38.61010",
- "datafu 32.05370",
- "datafu 32.05370",
- "hadoop 38.61010",
- "pig 96.10962",
- "pig 20.41135");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- * > t <- data.table(x=c("hadoop","bigdata","hadoop","datafu","bigdata","datafu","datafu","hadoop","pig","pig"),y=c(98.94791,38.61010,97.10575,32.05370,38.61010,32.05370,32.05370,38.61010,96.10962,20.41135))
- * > t <- t[order(x,y)]
- * > count<-c(2,3,1,1,1,1,1)
- * > library(entropy)
- * > entropy(count)
- * [1] 1.834372
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(1.834372);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- /**
- register $JAR_PATH
-
- define Entropy datafu.pig.stats.entropy.Entropy('$type', '$base');
-
- data = load 'input' as (val:double);
- --describe data;
- data_grouped = GROUP data ALL;
- data_out = FOREACH data_grouped {
- data_ordered = ORDER data BY val;
- GENERATE Entropy(data_ordered);
- };
- store data_out into 'output';
- */
- @Multiline private String logEntropy;
-
- @Test
- public void dupValEmpiricalEntropoyLog2Test() throws Exception
- {
- PigTest test = createPigTestFromString(logEntropy, "type=empirical", "base=log2");
-
- writeLinesToFile("input",
- "98.94791",
- "38.61010",
- "97.10575",
- "62.28313",
- "38.61010",
- "32.05370",
- "96.10962",
- "38.61010",
- "96.10962",
- "20.41135");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > v=c(98.94791,38.61010,97.10575,62.28313,38.61010,32.05370,96.10962,38.61010,96.10962,20.41135)
- * > table(v)
- * v
- * 20.41135 28.72388 32.0537 38.6101 38.8396 62.28313 96.10962 96.65888 97.10575 98.94791
- * 1 1 3 1 2 1 1
- * > count=c(1,1,3,1,2,1,1)
- * > freqs=count/sum(count)
- * > library(entropy)
- * > entropy(count, freqs, c("ML"), c("log2"))
- * [1] 2.646439
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(2.646439);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- @Test
- public void dupValEmpiricalEntropoyLog10Test() throws Exception
- {
- PigTest test = createPigTestFromString(logEntropy, "type=empirical", "base=log10");
-
- writeLinesToFile("input",
- "98.94791",
- "38.61010",
- "97.10575",
- "62.28313",
- "38.61010",
- "32.05370",
- "96.10962",
- "38.61010",
- "96.10962",
- "20.41135");
-
- test.runScript();
-
- /* Add expected values, computed using R:
- *
- * e.g.
- *
- * > v=c(98.94791,38.61010,97.10575,62.28313,38.61010,32.05370,96.10962,38.61010,96.10962,20.41135)
- * > table(v)
- * v
- * 20.41135 28.72388 32.0537 38.6101 38.8396 62.28313 96.10962 96.65888 97.10575 98.94791
- * 1 1 3 1 2 1 1
- * > count=c(1,1,3,1,2,1,1)
- * > freqs=count/sum(count)
- * > library(entropy)
- * > entropy(count, freqs, c("ML"), c("log10"))
- * [1] 0.7966576
- *
- */
- List<Double> expectedOutput = new ArrayList<Double>();
- expectedOutput.add(0.7966576);
-
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- verifyEqualEntropyOutput(expectedOutput, output, 5);
- }
-
- /**
- register $JAR_PATH
-
- define Entropy datafu.pig.stats.entropy.Entropy();
-
- data = load 'input' as (val:double);
- --describe data;
- data_grouped = GROUP data ALL;
- data_out = FOREACH data_grouped {
- GENERATE Entropy(data);
- };
- store data_out into 'output';
- */
- @Multiline private String noOrderEntropy;
-
- @Test
- public void noOrderEmpiricalEntropoyTest() throws Exception
- {
- PigTest test = createPigTestFromString(noOrderEntropy);
-
- writeLinesToFile("input",
- "98.94791",
- "38.61010",
- "38.61010",
- "37.10575",
- "62.28313",
- "38.61010",
- "32.05370",
- "96.10962",
- "38.61010",
- "96.10962",
- "20.41135");
-
- try {
- test.runScript();
- List<Tuple> output = this.getLinesForAlias(test, "data_out");
- fail( "Testcase should fail");
- } catch(Exception ex) {
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/text/NLPTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/text/NLPTests.java b/test/pig/datafu/test/pig/text/NLPTests.java
deleted file mode 100644
index 372b17d..0000000
--- a/test/pig/datafu/test/pig/text/NLPTests.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.text;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.test.pig.PigTests;
-
-
-public class NLPTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- define SentenceDetect datafu.pig.text.opennlp.SentenceDetect('data/en-sent.bin');
-
- data = LOAD 'input' AS (text: chararray);
-
- dump data;
-
- data2 = FOREACH data GENERATE SentenceDetect(text) AS sentences;
-
- dump data2;
-
- STORE data2 INTO 'output';
- */
- @Multiline
- private String sentenceDetectTest;
-
- @Test
- public void sentenceDetectTest() throws Exception
- {
- PigTest test = createPigTestFromString(sentenceDetectTest);
-
- writeLinesToFile("input",
- "This is a sentence. This is another sentence.",
- "Yet another sentence. One more just for luck.");
-
- assertOutput(test, "data2",
- "({(This is a sentence.),(This is another sentence.)})",
- "({(Yet another sentence.),(One more just for luck.)})");
- }
-
- /**
- register $JAR_PATH
-
- define TokenizeME datafu.pig.text.opennlp.TokenizeME('data/en-token.bin');
-
- data = LOAD 'input' AS (text: chararray);
-
- dump data;
-
- data2 = FOREACH data GENERATE TokenizeME(text) AS tokens;
-
- dump data2;
-
- STORE data2 INTO 'output';
- */
- @Multiline
- private String tokenizeMETest;
-
- @Test
- public void tokenizeMETest() throws Exception
- {
- PigTest test = createPigTestFromString(tokenizeMETest);
-
- writeLinesToFile("input",
- "This is a sentence. This is another sentence.",
- "Yet another sentence. One more just for luck.");
-
- assertOutput(test, "data2",
- "({(This),(is),(a),(sentence),(.),(This),(is),(another),(sentence),(.)})",
- "({(Yet),(another),(sentence),(.),(One),(more),(just),(for),(luck),(.)})");
- }
-
- /**
- register $JAR_PATH
-
- define TokenizeSimple datafu.pig.text.opennlp.TokenizeSimple();
-
- data = LOAD 'input' AS (text: chararray);
-
- dump data;
-
- data2 = FOREACH data GENERATE TokenizeSimple(text) AS tokens;
-
- dump data2;
-
- STORE data2 INTO 'output';
- */
- @Multiline
- private String tokenizeSimpleTest;
-
- @Test
- public void tokenizeSimpleTest() throws Exception
- {
- PigTest test = createPigTestFromString(tokenizeSimpleTest);
-
- writeLinesToFile("input",
- "This is a sentence. This is another sentence.",
- "Yet another sentence. One more just for luck.");
-
- assertOutput(test, "data2",
- "({(This),(is),(a),(sentence),(.),(This),(is),(another),(sentence),(.)})",
- "({(Yet),(another),(sentence),(.),(One),(more),(just),(for),(luck),(.)})");
- }
-
- /**
- register $JAR_PATH
-
- define TokenizeWhitespace datafu.pig.text.opennlp.TokenizeWhitespace();
-
- data = LOAD 'input' AS (text: chararray);
-
- dump data;
-
- data2 = FOREACH data GENERATE TokenizeWhitespace(text) AS tokens;
-
- dump data2;
-
- STORE data2 INTO 'output';
- */
- @Multiline
- private String tokenizeWhitespaceTest;
-
- @Test
- public void tokenizeWhitespaceTest() throws Exception
- {
- PigTest test = createPigTestFromString(tokenizeWhitespaceTest);
-
- writeLinesToFile("input",
- "This is a sentence. This is another sentence.",
- "Yet another sentence. One more just for luck.");
-
- assertOutput(test, "data2",
- "({(This),(is),(a),(sentence.),(This),(is),(another),(sentence.)})",
- "({(Yet),(another),(sentence.),(One),(more),(just),(for),(luck.)})");
- }
-
- /**
- register $JAR_PATH
-
- define TokenizeME datafu.pig.text.opennlp.TokenizeME('data/en-token.bin');
- define POSTag datafu.pig.text.opennlp.POSTag('data/en-pos-maxent.bin');
-
- data = LOAD 'input' AS (text: chararray);
-
- dump data;
-
- data2 = FOREACH data GENERATE TokenizeME(text) AS tokens;
-
- dump data2;
-
- data3 = FOREACH data2 GENERATE POSTag(tokens) as tagged;
-
- dump data3
-
- STORE data3 INTO 'output';
- */
- @Multiline
- private String POSTagTest;
-
- @Test
- public void POSTagTest() throws Exception
- {
- PigTest test = createPigTestFromString(POSTagTest);
-
- writeLinesToFile("input",
- "This is a sentence. This is another sentence.",
- "Yet another sentence. One more just for luck.");
-
- assertOutput(test, "data3",
- "({(This,DT,0.9649410482478001),(is,VBZ,0.9982592902509803),(a,DT,0.9967282012835504),(sentence,NN,0.9772619256460584),(.,.,0.4391067883074289),(This,DT,0.8346710130761914),(is,VBZ,0.9928885242823617),(another,DT,0.9761159923140399),(sentence,NN,0.9964463493238542),(.,.,0.9856037689871404)})",
- "({(Yet,RB,0.7638997090011364),(another,DT,0.9657669183153523),(sentence,NN,0.989193114719676),(.,.,0.20091718589945456),(One,CD,0.9229251494813668),(more,JJR,0.9360382000551335),(just,RB,0.8646324491545225),(for,IN,0.9851765355889605),(luck,NN,0.9883408827371651),(.,.,0.9746378518791978)})");
- }
-}
[11/19] DATAFU-27 Migrate build system to Gradle
Posted by wv...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/stats/VARTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/stats/VARTests.java b/datafu-pig/src/test/java/datafu/test/pig/stats/VARTests.java
new file mode 100644
index 0000000..cd59504
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/stats/VARTests.java
@@ -0,0 +1,521 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.stats;
+
+import static org.testng.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.pigunit.PigTest;
+import org.junit.Assert;
+import org.testng.annotations.Test;
+
+import datafu.pig.stats.DoubleVAR;
+import datafu.pig.stats.FloatVAR;
+import datafu.pig.stats.IntVAR;
+import datafu.pig.stats.LongVAR;
+import datafu.pig.stats.VAR;
+import datafu.test.pig.PigTests;
+
+public class VARTests extends PigTests
+{
+ /**
+
+
+ define VAR datafu.pig.stats.VAR();
+
+ data_in = LOAD 'input' as (val:$VAL_TYPE);
+ data_out = GROUP data_in ALL;
+ data_out = FOREACH data_out GENERATE VAR(data_in.val) AS variance;
+
+ --describe data_out;
+ STORE data_out into 'output';
+ */
+ @Multiline private String varTest;
+
+ @Test
+ public void varTestByteArray() throws Exception
+ {
+ PigTest test = createPigTestFromString(varTest,
+ "VAL_TYPE=bytearray");
+
+ String[] input = {"1","2","3","4","10","5","6","7","8","9"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(8.25)");
+ }
+
+ @Test
+ public void varTestDouble() throws Exception
+ {
+ PigTest test = createPigTestFromString(varTest,
+ "VAL_TYPE=double");
+
+ String[] input = {"1.0","2.0","3.0","4.0","10.0","5.0","6.0","7.0","8.0","9.0"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(8.25)");
+ }
+
+ @Test
+ public void varTestFloat() throws Exception
+ {
+ PigTest test = createPigTestFromString(varTest,
+ "VAL_TYPE=float");
+
+ String[] input = {"1.0","2.0","3.0","4.0","10.0","5.0","6.0","7.0","8.0","9.0"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(8.25)");
+ }
+
+ @Test
+ public void varTestInteger() throws Exception
+ {
+ PigTest test = createPigTestFromString(varTest,
+ "VAL_TYPE=int");
+
+ String[] input = {"1","2","3","4","10","5","6","7","8","9"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(8.25)");
+ }
+
+ @Test
+ public void varTestLong() throws Exception
+ {
+ PigTest test = createPigTestFromString(varTest,
+ "VAL_TYPE=long");
+
+ String[] input = {"1","2","3","4","10","5","6","7","8","9"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(8.25)");
+ }
+
+ @Test
+ public void varTestOneData() throws Exception
+ {
+ PigTest test = createPigTestFromString(varTest,
+ "VAL_TYPE=double");
+
+ String[] input = {"5.0"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(0.0)");
+ }
+
+
+ @Test
+ public void varTestZeroData() throws Exception
+ {
+ PigTest test = createPigTestFromString(varTest,
+ "VAL_TYPE=long");
+
+ String[] input = {};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),0);
+ }
+
+ @Test
+ public void varTestNullEntry() throws Exception
+ {
+ PigTest test = createPigTestFromString(varTest,
+ "VAL_TYPE=double");
+
+ String[] input = {"1","2","3","4","10","5","6","7","8","9","null"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(8.25)");
+ }
+
+ @Test
+ public void varTestNullEntries() throws Exception
+ {
+ PigTest test = createPigTestFromString(varTest,
+ "VAL_TYPE=float");
+
+ String[] input = {"1","2","3","4","10","5","6","7","8","9","null","null"};
+ writeLinesToFile("input", input);
+
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data_out", true);
+
+ assertEquals(output.size(),1);
+ assertEquals(output.get(0).toString(), "(8.25)");
+ }
+
+ @Test
+ public void varExecTest() throws Exception {
+ DoubleVAR var = new DoubleVAR();
+
+ DataBag bag;
+ Tuple input;
+ Double result;
+
+ bag = BagFactory.getInstance().newDefaultBag();
+ for (int i=1; i<=1000; i++)
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, (double)i);
+ bag.add(t);
+ }
+
+ input = TupleFactory.getInstance().newTuple(1);
+ input.set(0, bag);
+
+ result = var.exec(input);
+ Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
+
+ // do it again to check cleanup
+
+ bag = BagFactory.getInstance().newDefaultBag();
+ for (int i=1; i<=2000; i++)
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, (double)i);
+ bag.add(t);
+ }
+
+ input = TupleFactory.getInstance().newTuple(1);
+ input.set(0, bag);
+
+ result = var.exec(input);
+ Assert.assertTrue("Expected about 333333.2 but found " + result,Math.abs(333333.2 - result) < 1);
+ }
+
+ @Test
+ public void varAccumulateTest() throws Exception {
+ DoubleVAR var = new DoubleVAR();
+
+ Double result;
+
+ for (int i=1; i<=1000; i++)
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, (double)i);
+ DataBag bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(t);
+ Tuple input = TupleFactory.getInstance().newTuple(1);
+ input.set(0, bag);
+ var.accumulate(input);
+ }
+
+ result = var.getValue();
+ Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
+
+ // do it again to check cleanup
+ var.cleanup();
+
+ for (int i=1; i<=2000; i++)
+ {
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, (double)i);
+ DataBag bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(t);
+ Tuple input = TupleFactory.getInstance().newTuple(1);
+ input.set(0, bag);
+ var.accumulate(input);
+ }
+
+ result = var.getValue();
+ Assert.assertTrue("Expected about 333333.2 but found " + result,Math.abs(333333.2 - result) < 1);
+ }
+
+ // make sure intermediate works, where initial just passes through a single tuple, and intermediate receives a large bag of the resulting tuples
+ @Test
+ public void varDoubleAlgebraicIntermediateTest() throws Exception {
+ DoubleVAR.Initial initialVar = new DoubleVAR.Initial();
+ DoubleVAR.Intermediate intermediateVar = new DoubleVAR.Intermediate();
+ DoubleVAR.Final finalVar = new DoubleVAR.Final();
+
+
+ DataBag intermediateBag = BagFactory.getInstance().newDefaultBag();
+
+ for (int i=1; i<=1000; i++)
+ {
+ DataBag bag;
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, (double)i);
+ bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(t);
+ Tuple input = TupleFactory.getInstance().newTuple(bag);
+ Tuple intermediateTuple = initialVar.exec(input);
+ intermediateBag.add(intermediateTuple);
+ }
+
+ Tuple intermediateTuple = intermediateVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
+ intermediateBag = BagFactory.getInstance().newDefaultBag(Arrays.asList(intermediateTuple));
+ Double result = finalVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
+
+ Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
+ }
+
+ // make sure final works, where initial just passes through a single tuple, intermediate does the same, and final receives the remainder
+ @Test
+ public void varDoubleAlgebraicFinalTest() throws Exception {
+ DoubleVAR.Initial initialVar = new DoubleVAR.Initial();
+ DoubleVAR.Intermediate intermediateVar = new DoubleVAR.Intermediate();
+ DoubleVAR.Final finalVar = new DoubleVAR.Final();
+
+ DataBag finalBag = BagFactory.getInstance().newDefaultBag();
+
+ for (int i=1; i<=1000; i++)
+ {
+ DataBag bag;
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, (double)i);
+ bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(t);
+ Tuple input = TupleFactory.getInstance().newTuple(bag);
+ Tuple intermediateTuple = initialVar.exec(input);
+ DataBag intermediateBag = BagFactory.getInstance().newDefaultBag();
+ intermediateBag.add(intermediateTuple);
+ intermediateTuple = intermediateVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
+ finalBag.add(intermediateTuple);
+ }
+
+ Double result = finalVar.exec(TupleFactory.getInstance().newTuple(finalBag));
+
+ Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
+ }
+
+ // make sure intermediate works, where initial just passes through a single tuple, and intermediate receives a large bag of the resulting tuples
+ @Test
+ public void varFloatAlgebraicIntermediateTest() throws Exception {
+ FloatVAR.Initial initialVar = new FloatVAR.Initial();
+ FloatVAR.Intermediate intermediateVar = new FloatVAR.Intermediate();
+ FloatVAR.Final finalVar = new FloatVAR.Final();
+
+
+ DataBag intermediateBag = BagFactory.getInstance().newDefaultBag();
+
+ for (int i=1; i<=1000; i++)
+ {
+ DataBag bag;
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, (float)i);
+ bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(t);
+ Tuple input = TupleFactory.getInstance().newTuple(bag);
+ Tuple intermediateTuple = initialVar.exec(input);
+ intermediateBag.add(intermediateTuple);
+ }
+
+ Tuple intermediateTuple = intermediateVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
+ intermediateBag = BagFactory.getInstance().newDefaultBag(Arrays.asList(intermediateTuple));
+ Double result = finalVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
+
+ Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
+ }
+
+ // make sure final works, where initial just passes through a single tuple, intermediate does the same, and final receives the remainder
+ @Test
+ public void varFloatAlgebraicFinalTest() throws Exception {
+ FloatVAR.Initial initialVar = new FloatVAR.Initial();
+ FloatVAR.Intermediate intermediateVar = new FloatVAR.Intermediate();
+ FloatVAR.Final finalVar = new FloatVAR.Final();
+
+ DataBag finalBag = BagFactory.getInstance().newDefaultBag();
+
+ for (int i=1; i<=1000; i++)
+ {
+ DataBag bag;
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, (float)i);
+ bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(t);
+ Tuple input = TupleFactory.getInstance().newTuple(bag);
+ Tuple intermediateTuple = initialVar.exec(input);
+ DataBag intermediateBag = BagFactory.getInstance().newDefaultBag();
+ intermediateBag.add(intermediateTuple);
+ intermediateTuple = intermediateVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
+ finalBag.add(intermediateTuple);
+ }
+
+ Double result = finalVar.exec(TupleFactory.getInstance().newTuple(finalBag));
+
+ Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
+ }
+
+ // make sure intermediate works, where initial just passes through a single tuple, and intermediate receives a large bag of the resulting tuples
+ @Test
+ public void varIntAlgebraicIntermediateTest() throws Exception {
+ IntVAR.Initial initialVar = new IntVAR.Initial();
+ IntVAR.Intermediate intermediateVar = new IntVAR.Intermediate();
+ IntVAR.Final finalVar = new IntVAR.Final();
+
+
+ DataBag intermediateBag = BagFactory.getInstance().newDefaultBag();
+
+ for (int i=1; i<=1000; i++)
+ {
+ DataBag bag;
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, (int)i);
+ bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(t);
+ Tuple input = TupleFactory.getInstance().newTuple(bag);
+ Tuple intermediateTuple = initialVar.exec(input);
+ intermediateBag.add(intermediateTuple);
+ }
+
+ Tuple intermediateTuple = intermediateVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
+ intermediateBag = BagFactory.getInstance().newDefaultBag(Arrays.asList(intermediateTuple));
+ Double result = finalVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
+
+ Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
+ }
+
+ // make sure final works, where initial just passes through a single tuple, intermediate does the same, and final receives the remainder
+ @Test
+ public void varIntAlgebraicFinalTest() throws Exception {
+ IntVAR.Initial initialVar = new IntVAR.Initial();
+ IntVAR.Intermediate intermediateVar = new IntVAR.Intermediate();
+ IntVAR.Final finalVar = new IntVAR.Final();
+
+ DataBag finalBag = BagFactory.getInstance().newDefaultBag();
+
+ for (int i=1; i<=1000; i++)
+ {
+ DataBag bag;
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, (int)i);
+ bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(t);
+ Tuple input = TupleFactory.getInstance().newTuple(bag);
+ Tuple intermediateTuple = initialVar.exec(input);
+ DataBag intermediateBag = BagFactory.getInstance().newDefaultBag();
+ intermediateBag.add(intermediateTuple);
+ intermediateTuple = intermediateVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
+ finalBag.add(intermediateTuple);
+ }
+
+ Double result = finalVar.exec(TupleFactory.getInstance().newTuple(finalBag));
+
+ Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
+ }
+
+ // make sure intermediate works, where initial just passes through a single tuple, and intermediate receives a large bag of the resulting tuples
+ @Test
+ public void varLongAlgebraicIntermediateTest() throws Exception {
+ LongVAR.Initial initialVar = new LongVAR.Initial();
+ LongVAR.Intermediate intermediateVar = new LongVAR.Intermediate();
+ LongVAR.Final finalVar = new LongVAR.Final();
+
+
+ DataBag intermediateBag = BagFactory.getInstance().newDefaultBag();
+
+ for (int i=1; i<=1000; i++)
+ {
+ DataBag bag;
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, (long)i);
+ bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(t);
+ Tuple input = TupleFactory.getInstance().newTuple(bag);
+ Tuple intermediateTuple = initialVar.exec(input);
+ intermediateBag.add(intermediateTuple);
+ }
+
+ Tuple intermediateTuple = intermediateVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
+ intermediateBag = BagFactory.getInstance().newDefaultBag(Arrays.asList(intermediateTuple));
+ Double result = finalVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
+
+ Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
+ }
+
+ // make sure final works, where initial just passes through a single tuple, intermediate does the same, and final receives the remainder
+ @Test
+ public void varLongAlgebraicFinalTest() throws Exception {
+ LongVAR.Initial initialVar = new LongVAR.Initial();
+ LongVAR.Intermediate intermediateVar = new LongVAR.Intermediate();
+ LongVAR.Final finalVar = new LongVAR.Final();
+
+ DataBag finalBag = BagFactory.getInstance().newDefaultBag();
+
+ for (int i=1; i<=1000; i++)
+ {
+ DataBag bag;
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ t.set(0, (long)i);
+ bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(t);
+ Tuple input = TupleFactory.getInstance().newTuple(bag);
+ Tuple intermediateTuple = initialVar.exec(input);
+ DataBag intermediateBag = BagFactory.getInstance().newDefaultBag();
+ intermediateBag.add(intermediateTuple);
+ intermediateTuple = intermediateVar.exec(TupleFactory.getInstance().newTuple(intermediateBag));
+ finalBag.add(intermediateTuple);
+ }
+
+ Double result = finalVar.exec(TupleFactory.getInstance().newTuple(finalBag));
+
+ Assert.assertTrue("Expected about 83333.25 but found " + result,Math.abs(83333.25 - result) < 0.0001);
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/stats/WilsonBinConfTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/stats/WilsonBinConfTests.java b/datafu-pig/src/test/java/datafu/test/pig/stats/WilsonBinConfTests.java
new file mode 100644
index 0000000..b888595
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/stats/WilsonBinConfTests.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.stats;
+
+import static org.testng.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.test.pig.PigTests;
+
+public class WilsonBinConfTests extends PigTests
+{
+ /**
+
+
+ define WilsonBinConf datafu.pig.stats.WilsonBinConf('$alpha');
+
+ data = load 'input' as (successes:long, totals:long);
+ --describe data;
+
+ data_out = FOREACH data GENERATE WilsonBinConf(successes, totals) as interval;
+ data_out = FOREACH data_out GENERATE FLATTEN(interval);
+
+ store data_out into 'output';
+ */
+ @Multiline private String wilsonBinConf;
+
+ @Test
+ public void wilsonTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(wilsonBinConf,
+ "alpha=0.05"); // alpha is 0.05 for 95% confidence
+
+ writeLinesToFile("input",
+ "1\t1",
+ "1\t2",
+ "50\t100",
+ "500\t1000",
+ "999\t1000",
+ "1000\t1000",
+ "998\t1000");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * library(Hmisc)
+ *
+ * binconf(50,100)
+ * binconf(500,1000)
+ *
+ */
+ List<String> expectedOutput = new ArrayList<String>();
+ expectedOutput.add("0.05129,1.00000");
+ expectedOutput.add("0.02565,0.97435");
+ expectedOutput.add("0.40383,0.59617");
+ expectedOutput.add("0.46907,0.53093");
+ expectedOutput.add("0.99436,0.99995");
+ expectedOutput.add("0.99617,1.00000");
+ expectedOutput.add("0.99274,0.99945");
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ Iterator<String> expectationIterator = expectedOutput.iterator();
+ for (Tuple t : output)
+ {
+ assertTrue(expectationIterator.hasNext());
+ Double lower = (Double)t.get(0);
+ Double upper = (Double)t.get(1);
+ assertEquals(String.format("%.5f,%.5f",lower,upper),expectationIterator.next());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/AbstractEntropyTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/AbstractEntropyTests.java b/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/AbstractEntropyTests.java
new file mode 100644
index 0000000..6e512d4
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/AbstractEntropyTests.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.stats.entropy;
+
+import static org.testng.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.pig.data.Tuple;
+
+import org.apache.pig.backend.executionengine.ExecException;
+
+import datafu.test.pig.PigTests;
+
+public abstract class AbstractEntropyTests extends PigTests
+{
+ protected void verifyEqualEntropyOutput(List<Double> expectedOutput, List<Tuple> output, int digits) throws ExecException {
+ assertEquals(expectedOutput.size(), output.size());
+ Iterator<Double> expectationIterator = expectedOutput.iterator();
+ String formatDigits = "%." + digits + "f";
+ for (Tuple t : output)
+ {
+ Double entropy = (Double)t.get(0);
+ assertEquals(String.format(formatDigits,entropy),String.format(formatDigits, expectationIterator.next()));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/ChaoShenEntropyTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/ChaoShenEntropyTests.java b/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/ChaoShenEntropyTests.java
new file mode 100644
index 0000000..c20eada
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/ChaoShenEntropyTests.java
@@ -0,0 +1,371 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.stats.entropy;
+
+import static org.testng.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.test.pig.PigTests;
+
+/*
+ * R's entropy library: http://cran.r-project.org/web/packages/entropy/entropy.pdf
+ * used as our test benchmark
+ */
+public class ChaoShenEntropyTests extends AbstractEntropyTests
+{
+ /**
+
+ define Entropy datafu.pig.stats.entropy.Entropy('$type','$base');
+
+ data = load 'input' as (val:double);
+ --describe data;
+ data_grouped = GROUP data ALL;
+ data_out = FOREACH data_grouped {
+ data_ordered = ORDER data BY val;
+ GENERATE Entropy(data_ordered);
+ };
+ store data_out into 'output';
+ */
+ @Multiline private String entropy;
+
+ @Test
+ public void uniqValChaoShenEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(entropy, "type=chaosh", "base=log");
+
+ writeLinesToFile("input",
+ "98.94791",
+ "38.61010",
+ "97.10575",
+ "62.28313",
+ "38.83960",
+ "32.05370",
+ "96.10962",
+ "28.72388",
+ "96.65888",
+ "20.41135");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > v=c(98.94791,38.61010,97.10575,62.28313,38.83960,32.05370,96.10962,28.72388,96.65888,20.41135)
+ * > table(v)
+ * v
+ * 20.41135 28.72388 32.0537 38.6101 38.8396 62.28313 96.10962 96.65888 97.10575 98.94791
+ * 1 1 1 1 1 1 1 1 1 1
+ * > count=c(1,1,1,1,1,1,1,1,1,1)
+ * > library(entropy)
+ * > entropy(count,count/sum(count),c("CS"),c("log"))
+ * [1] 4.816221
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(4.816221);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void singleValChaoShenEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(entropy, "type=chaosh", "base=log");
+
+ writeLinesToFile("input",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > v=c(98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791)
+ * > table(v)
+ * v
+ * 98.94791
+ * 10
+ * > count=(10)
+ * > entropy(count,count/sum(count),c("CS"),c("log"))
+ * [1] 0
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(0.0);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void dupValChaoShenEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(entropy,"type=chaosh", "base=log");
+
+ writeLinesToFile("input",
+ "98.94791",
+ "38.61010",
+ "97.10575",
+ "62.28313",
+ "38.61010",
+ "32.05370",
+ "96.10962",
+ "38.61010",
+ "96.10962",
+ "20.41135");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > v=c(98.94791,38.61010,97.10575,62.28313,38.61010,32.05370,96.10962,38.61010,96.10962,20.41135)
+ * > table(v)
+ * v
+ * 20.41135 28.72388 32.0537 38.6101 38.8396 62.28313 96.10962 96.65888 97.10575 98.94791
+ * 1 1 3 1 2 1 1
+ * > count=c(1,1,3,1,2,1,1)
+ * > library(entropy)
+ * > entropy(count,count/sum(count),c("CS"),c("log"))
+ * [1] 2.57429
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(2.57429);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+
+ @Test
+ public void emptyInputBagChaoShenEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(entropy, "type=chaosh", "base=log");
+
+ writeLinesToFile("input"
+ );
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > v=c()
+ * > table(v)
+ * < table of extent 0 >
+ * > count=c()
+ * > library(entropy)
+ * > entropy(count,count/sum(count),c("CS"),c("log"))
+ * [1] 0
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void singleElemInputBagChaoShenEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(entropy, "type=chaosh", "base=log");
+
+ writeLinesToFile("input",
+ "98.94791");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > count=c(1)
+ * > library(entropy)
+ * > entropy(count,count/sum(count),c("CS"),c("log"))
+ * [1] 0
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(0.0);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ /**
+
+ define Entropy datafu.pig.stats.entropy.Entropy('$type','$base');
+
+ data = load 'input' as (x:chararray, y:double);
+ --describe data;
+ data_grouped = GROUP data ALL;
+ data_out = FOREACH data_grouped {
+ data_ordered = ORDER data BY x,y;
+ GENERATE Entropy(data_ordered);
+ };
+ store data_out into 'output';
+ */
+ @Multiline private String pairEntropy;
+
+ @Test
+ public void dupPairValChaoShenEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(pairEntropy, "type=chaosh", "base=log");
+
+ writeLinesToFile("input",
+ "hadoop 98.94791",
+ "bigdata 38.61010",
+ "hadoop 97.10575",
+ "datafu 32.05370",
+ "bigdata 38.61010",
+ "datafu 32.05370",
+ "datafu 32.05370",
+ "hadoop 38.61010",
+ "pig 96.10962",
+ "pig 20.41135");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ * > t <- data.table(x=c("hadoop","bigdata","hadoop","datafu","bigdata","datafu","datafu","hadoop","pig","pig"),y=c(98.94791,38.61010,97.10575,32.05370,38.61010,32.05370,32.05370,38.61010,96.10962,20.41135))
+ * > t <- t[order(x,y)]
+ * > count<-c(2,3,1,1,1,1,1)
+ * > library(entropy)
+ * > entropy(count,count/sum(count),c("CS"),c("log"))
+ * [1] 2.57429
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(2.57429);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void dupValChaoShenEntropoyLog2Test() throws Exception
+ {
+ PigTest test = createPigTestFromString(entropy,"type=chaosh", "base=log2");
+
+ writeLinesToFile("input",
+ "98.94791",
+ "38.61010",
+ "97.10575",
+ "62.28313",
+ "38.61010",
+ "32.05370",
+ "96.10962",
+ "38.61010",
+ "96.10962",
+ "20.41135");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > v=c(98.94791,38.61010,97.10575,62.28313,38.61010,32.05370,96.10962,38.61010,96.10962,20.41135)
+ * > table(v)
+ * v
+ * 20.41135 28.72388 32.0537 38.6101 38.8396 62.28313 96.10962 96.65888 97.10575 98.94791
+ * 1 1 3 1 2 1 1
+ * > count=c(1,1,3,1,2,1,1)
+ * > freqs=count/sum(count)
+ * > library(entropy)
+ * > entropy(count,count/sum(count),c("CS"),c("log2"))
+ * [1] 3.713915
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(3.713915);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void dupValChaoShenEntropoyLog10Test() throws Exception
+ {
+ PigTest test = createPigTestFromString(entropy, "type=chaosh", "base=log10");
+
+ writeLinesToFile("input",
+ "98.94791",
+ "38.61010",
+ "97.10575",
+ "62.28313",
+ "38.61010",
+ "32.05370",
+ "96.10962",
+ "38.61010",
+ "96.10962",
+ "20.41135");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > v=c(98.94791,38.61010,97.10575,62.28313,38.61010,32.05370,96.10962,38.61010,96.10962,20.41135)
+ * > table(v)
+ * v
+ * 20.41135 28.72388 32.0537 38.6101 38.8396 62.28313 96.10962 96.65888 97.10575 98.94791
+ * 1 1 3 1 2 1 1
+ * > count=c(1,1,3,1,2,1,1)
+ * > library(entropy)
+ * > entropy(count,count/sum(count),c("CS"),c("log10"))
+ * [1] 1.118
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(1.118);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/CondEntropyTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/CondEntropyTests.java b/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/CondEntropyTests.java
new file mode 100644
index 0000000..0587ed1
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/CondEntropyTests.java
@@ -0,0 +1,407 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.stats.entropy;
+
+import static org.testng.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.test.pig.PigTests;
+
+
+/*
+ * Use R function to compute condition entropy as the test benchmark
+ * http://cran.r-project.org/web/packages/infotheo/infotheo.pdf
+ */
+public class CondEntropyTests extends AbstractEntropyTests
+{
+ /**
+
+ define CondEntropy datafu.pig.stats.entropy.CondEntropy();
+
+ data = load 'input' as (valX:double, valY:chararray);
+ --describe data;
+ data_grouped = GROUP data ALL;
+ data_out = FOREACH data_grouped {
+ data_ordered = ORDER data BY *;
+ GENERATE CondEntropy(data_ordered);
+ };
+ store data_out into 'output';
+ */
+ @Multiline private String condEntropy;
+
+ @Test
+ public void uniqValEmpiricalCondEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(condEntropy);
+
+ writeLinesToFile("input",
+ "98.94791 click",
+ "38.61010 view",
+ "97.10575 view",
+ "62.28313 click",
+ "38.83960 click",
+ "32.05370 view",
+ "96.10962 view",
+ "28.72388 click",
+ "96.65888 view",
+ "20.41135 click");
+
+ test.runScript();
+
+ /*
+ * library(infotheo)
+ * X=c("98.94791","38.61010","97.10575","62.28313","38.83960","32.05370","96.10962","28.72388","96.65888","20.41135")
+ * Y=c("click","view","view","click","click","view","view","click","view","click")
+ * condentropy(Y,X)
+ * [1] 0
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(0.0);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void singleValEmpiricalCondEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(condEntropy);
+
+ writeLinesToFile("input",
+ "98.94791 click",
+ "98.94791 click",
+ "98.94791 click",
+ "98.94791 click",
+ "98.94791 click",
+ "98.94791 click",
+ "98.94791 click",
+ "98.94791 click",
+ "98.94791 click",
+ "98.94791 click");
+
+ test.runScript();
+
+ /*
+ * library(infotheo)
+ * X=c("98.94791","98.94791","98.94791","98.94791","98.94791","98.94791","98.94791","98.94791","98.94791","98.94791")
+ * Y=c("click","click","click","click","click","click","click","click","click","click")
+ * condentropy(Y,X)
+ * [1] 0
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(0.0);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void dupValEmpiricalCondEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(condEntropy);
+
+ writeLinesToFile("input",
+ "98.94791 click",
+ "38.61010 click",
+ "97.10575 view",
+ "62.28313 view",
+ "38.61010 view",
+ "32.05370 view",
+ "96.10962 click",
+ "38.61010 click",
+ "96.10962 view",
+ "20.41135 click");
+
+ test.runScript();
+
+ /*
+ * library(infotheo)
+ * X=c("98.94791","38.61010","97.10575","62.28313","38.61010","32.05370","96.10962","38.61010","96.10962","20.41135")
+ * Y=c("click","click","view","view","view","view","click","click","view","click")
+ * condentropy(Y,X)
+ * [1] 0.3295837
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(0.3295837);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void emptyInputBagEmpiricalCondEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(condEntropy);
+
+ writeLinesToFile("input"
+ );
+
+ test.runScript();
+
+ List<Double> expectedOutput = new ArrayList<Double>();
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void singleElemInputBagEmpiricalCondEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(condEntropy);
+
+ writeLinesToFile("input",
+ "98.94791 view");
+
+ test.runScript();
+ /*
+ * library(infotheo)
+ * X = c("98.94791")
+ * Y = c("view")
+ * condentropy(Y,X)
+ * [1] 0
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(0.0);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ /**
+
+ define Entropy datafu.pig.stats.entropy.CondEntropy();
+
+ data = load 'input' as (valX1:chararray, valX2:chararray, valY:chararray);
+ data = foreach data generate (valX1, valX2) as X, valY as Y;
+ --describe data;
+ data_grouped = GROUP data ALL;
+ data_out = FOREACH data_grouped {
+ data_ordered = ORDER data BY *;
+ GENERATE Entropy(data_ordered);
+ };
+
+ store data_out into 'output';
+ */
+ @Multiline private String pairCondEntropy;
+
+ @Test
+ public void dupPairValEmpiricalCondEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(pairCondEntropy);
+
+ writeLinesToFile("input",
+ "hadoop bigdata click",
+ "hadoop pig view",
+ "hadoop datafu click",
+ "datafu pig click",
+ "bigdata pig view",
+ "datafu pig click",
+ "datafu pig view",
+ "hadoop bigdata view",
+ "pig datafu view",
+ "pig datafu view");
+
+ test.runScript();
+
+ /*
+ * library(infotheo)
+ * X=c("hadoop bigdata","hadoop pig","hadoop datafu","datafu pig","bigdata pig","datafu pig","datafu pig","hadoop bigdata","pig datafu","pig datafu")
+ * Y=c("click","view","click","click","view","click","view","view","view","view")
+ * condentropy(X,Y)
+ * [1] 0.3295837
+ */
+
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(0.3295837);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ /**
+
+ define CondEntropy datafu.pig.stats.entropy.CondEntropy('$type','$base');
+
+ data = load 'input' as (valX:double, valY:chararray);
+ --describe data;
+ data_grouped = GROUP data ALL;
+ data_out = FOREACH data_grouped {
+ data_ordered = ORDER data BY *;
+ GENERATE CondEntropy(data_ordered);
+ };
+ store data_out into 'output';
+ */
+ @Multiline private String condLogEntropy;
+
+ @Test
+ public void dupValEmpiricalCondEntropoyLog2Test() throws Exception
+ {
+ PigTest test = createPigTestFromString(condLogEntropy, "type=empirical", "base=log2");
+
+ writeLinesToFile("input",
+ "98.94791 click",
+ "38.61010 click",
+ "97.10575 view",
+ "62.28313 view",
+ "38.61010 view",
+ "32.05370 view",
+ "96.10962 click",
+ "38.61010 click",
+ "96.10962 view",
+ "20.41135 click");
+
+ test.runScript();
+
+ /*
+ * library(infotheo)
+ * X=c("98.94791","38.61010","97.10575","62.28313","38.61010","32.05370","96.10962","38.61010","96.10962","20.41135")
+ * Y=c("click","click","view","view","view","view","click","click","view","click")
+ * condentropy(Y,X)/log(2)
+ * [1] 0.4754888
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(0.4754888);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void dupValEmpiricalCondEntropoyLog10Test() throws Exception
+ {
+ PigTest test = createPigTestFromString(condLogEntropy, "type=empirical", "base=log10");
+
+ writeLinesToFile("input",
+ "98.94791 click",
+ "38.61010 click",
+ "97.10575 view",
+ "62.28313 view",
+ "38.61010 view",
+ "32.05370 view",
+ "96.10962 click",
+ "38.61010 click",
+ "96.10962 view",
+ "20.41135 click");
+
+ test.runScript();
+
+ /*
+ * library(infotheo)
+ * X=c("98.94791","38.61010","97.10575","62.28313","38.61010","32.05370","96.10962","38.61010","96.10962","20.41135")
+ * Y=c("click","click","view","view","view","view","click","click","view","click")
+ * condentropy(Y,X)/log(10)
+ * [1] 0.1431364
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(0.1431364);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ /**
+
+ define CondEntropy datafu.pig.stats.entropy.CondEntropy();
+
+ data = load 'input' as (valX:double, valY:chararray);
+ --describe data;
+ data_grouped = GROUP data ALL;
+ data_out = FOREACH data_grouped {
+ GENERATE CondEntropy(data);
+ };
+ store data_out into 'output';
+ */
+ @Multiline private String noOrderCondEntropy;
+
+ @Test
+ public void noOrderEmpiricalCondEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(noOrderCondEntropy);
+
+ writeLinesToFile("input",
+ "98.94791 click",
+ "38.61010 view",
+ "97.10575 view",
+ "62.28313 click",
+ "38.83960 click",
+ "32.05370 view",
+ "96.10962 view",
+ "28.72388 click",
+ "96.65888 view",
+ "20.41135 click");
+
+ try {
+ test.runScript();
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ fail( "Testcase should fail");
+ } catch(Exception ex) {}
+ }
+
+ /**
+
+ define CondEntropy datafu.pig.stats.entropy.CondEntropy();
+
+ data = load 'input' as (valX:double);
+ --describe data;
+ data_grouped = GROUP data ALL;
+ data_out = FOREACH data_grouped {
+ data_ordered = ORDER data BY *;
+ GENERATE CondEntropy(data_ordered);
+ };
+ store data_out into 'output';
+ */
+ @Multiline private String invalidInputCondEntropy;
+
+ @Test
+ public void invalidInputEmpiricalCondEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(invalidInputCondEntropy);
+
+ writeLinesToFile("input",
+ "98.94791",
+ "38.61010",
+ "97.10575",
+ "62.28313",
+ "38.83960",
+ "32.05370",
+ "96.10962",
+ "28.72388",
+ "96.65888",
+ "20.41135");
+
+ try {
+ test.runScript();
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ fail( "Testcase should fail");
+ } catch(Exception ex) {
+ assertTrue(ex.getMessage().indexOf("The field schema of the input tuple is null or its size is not 2") >= 0);
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/EmpiricalCountEntropyTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/EmpiricalCountEntropyTests.java b/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/EmpiricalCountEntropyTests.java
new file mode 100644
index 0000000..594b91b
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/EmpiricalCountEntropyTests.java
@@ -0,0 +1,578 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.stats.entropy;
+
+import static org.testng.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.test.pig.PigTests;
+
+/*
+ * R's entropy library: http://cran.r-project.org/web/packages/entropy/entropy.pdf
+ * used as our test benchmark
+ */
+public class EmpiricalCountEntropyTests extends AbstractEntropyTests
+{
+ /**
+
+ define Entropy datafu.pig.stats.entropy.EmpiricalCountEntropy();
+
+ data = load 'input' as (val:double);
+ --describe data;
+ data_grouped = GROUP data BY val;
+ data_cnt = FOREACH data_grouped GENERATE COUNT(data) AS cnt;
+ data_cnt_grouped = GROUP data_cnt ALL;
+ data_out = FOREACH data_cnt_grouped GENERATE Entropy(data_cnt);
+ store data_out into 'output';
+ */
+ @Multiline private String entropy;
+
+ @Test
+ public void uniqValEntropyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(entropy);
+
+ writeLinesToFile("input",
+ "98.94791",
+ "38.61010",
+ "97.10575",
+ "62.28313",
+ "38.83960",
+ "32.05370",
+ "96.10962",
+ "28.72388",
+ "96.65888",
+ "20.41135");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > v=c(98.94791,38.61010,97.10575,62.28313,38.83960,32.05370,96.10962,28.72388,96.65888,20.41135)
+ * > table(v)
+ * v
+ * 20.41135 28.72388 32.0537 38.6101 38.8396 62.28313 96.10962 96.65888 97.10575 98.94791
+ * 1 1 1 1 1 1 1 1 1 1
+ * > count=c(1,1,1,1,1,1,1,1,1,1)
+ * > library(entropy)
+ * > entropy(count)
+ * [1] 2.302585
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(2.302585);
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void singleValEntropyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(entropy);
+
+ writeLinesToFile("input",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > v=c(98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791)
+ * > table(v)
+ * v
+ * 98.94791
+ * 10
+ * > count=(10)
+ * > library(entropy)
+ * > entropy(count)
+ * [1] 0
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(0.0);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void dupValEntropyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(entropy);
+
+ writeLinesToFile("input",
+ "98.94791",
+ "38.61010",
+ "97.10575",
+ "62.28313",
+ "38.61010",
+ "32.05370",
+ "96.10962",
+ "38.61010",
+ "96.10962",
+ "20.41135");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > v=c(98.94791,38.61010,97.10575,62.28313,38.61010,32.05370,96.10962,38.61010,96.10962,20.41135)
+ * > table(v)
+ * v
+ * 20.41135 32.0537 38.6101 62.28313 96.10962 97.10575 98.94791
+ * 1 1 3 1 2 1 1
+ * > count=c(1,1,3,1,2,1,1)
+ * > library(entropy)
+ * > entropy(count)
+ * [1] 1.834372
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(1.834372);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void emptyInputBagEntropyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(entropy);
+
+ writeLinesToFile("input"
+ );
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > v=c()
+ * > table(v)
+ * < table of extent 0 >
+ * > count=c()
+ * > library(entropy)
+ * > entropy(count)
+ * [1] 0
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void singleElemInputBagEntropyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(entropy);
+
+ writeLinesToFile("input",
+ "98.94791");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > count=c(1)
+ * > library(entropy)
+ * > entropy(count)
+ * [1] 0
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(0.0);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ /**
+
+ define Entropy datafu.pig.stats.entropy.EmpiricalCountEntropy('$base');
+
+ data = load 'input' as (x:chararray, y:double);
+ --describe data;
+ data_grouped = GROUP data BY (x, y);
+ data_cnt = FOREACH data_grouped GENERATE COUNT(data);
+ data_cnt_grouped = GROUP data_cnt ALL;
+ data_out = FOREACH data_cnt_grouped GENERATE Entropy(data_cnt);
+ store data_out into 'output';
+ */
+ @Multiline private String pairLogEntropy;
+
+ @Test
+ public void dupPairValEntropyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(pairLogEntropy, "base=log");
+
+ writeLinesToFile("input",
+ "hadoop 98.94791",
+ "bigdata 38.61010",
+ "hadoop 97.10575",
+ "datafu 32.05370",
+ "bigdata 38.61010",
+ "datafu 32.05370",
+ "datafu 32.05370",
+ "hadoop 38.61010",
+ "pig 96.10962",
+ "pig 20.41135");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ * > t <- data.table(x=c("hadoop","bigdata","hadoop","datafu","bigdata","datafu","datafu","hadoop","pig","pig"),y=c(98.94791,38.61010,97.10575,32.05370,38.61010,32.05370,32.05370,38.61010,96.10962,20.41135))
+ * > t <- t[order(x,y)]
+ * > count<-c(2,3,1,1,1,1,1)
+ * > library(entropy)
+ * > entropy(count)
+ * [1] 1.834372
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(1.834372);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ /**
+
+ define Entropy datafu.pig.stats.entropy.EmpiricalCountEntropy('$base');
+
+ data = load 'input' as (val:double);
+ --describe data;
+ data_grouped = GROUP data BY val;
+ data_cnt = FOREACH data_grouped GENERATE COUNT(data) AS cnt;
+ data_cnt_grouped = GROUP data_cnt ALL;
+ data_out = FOREACH data_cnt_grouped GENERATE Entropy(data_cnt);
+ store data_out into 'output';
+ */
+ @Multiline private String logEntropy;
+
+ @Test
+ public void dupValEntropyLog2Test() throws Exception
+ {
+ PigTest test = createPigTestFromString(logEntropy, "base=log2");
+
+ writeLinesToFile("input",
+ "98.94791",
+ "38.61010",
+ "97.10575",
+ "62.28313",
+ "38.61010",
+ "32.05370",
+ "96.10962",
+ "38.61010",
+ "96.10962",
+ "20.41135");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > v=c(98.94791,38.61010,97.10575,62.28313,38.61010,32.05370,96.10962,38.61010,96.10962,20.41135)
+ * > table(v)
+ * v
+ * 20.41135 32.0537 38.6101 62.28313 96.10962 97.10575 98.94791
+ * 1 1 3 1 2 1 1
+ * > count=c(1,1,3,1,2,1,1)
+ * > library(entropy)
+ * > entropy(count, count/sum(count), c("ML"),c("log2"))
+ * [1] 2.646439
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(2.646439);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void dupValEntropyLog10Test() throws Exception
+ {
+ PigTest test = createPigTestFromString(logEntropy, "base=log10");
+
+ writeLinesToFile("input",
+ "98.94791",
+ "38.61010",
+ "97.10575",
+ "62.28313",
+ "38.61010",
+ "32.05370",
+ "96.10962",
+ "38.61010",
+ "96.10962",
+ "20.41135");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > v=c(98.94791,38.61010,97.10575,62.28313,38.61010,32.05370,96.10962,38.61010,96.10962,20.41135)
+ * > table(v)
+ * v
+ * 20.41135 32.0537 38.6101 62.28313 96.10962 97.10575 98.94791
+ * 1 1 3 1 2 1 1
+ * > count=c(1,1,3,1,2,1,1)
+ * > library(entropy)
+ * > entropy(count, count/sum(count), c("ML"),c("log10"))
+ * [1] 0.7966576
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(0.7966576);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+
+ /**
+
+ define Entropy datafu.pig.stats.entropy.EmpiricalCountEntropy();
+
+ data_cnt = load 'input' as (val:int);
+ --describe data_cnt;
+ data_cnt_grouped = GROUP data_cnt ALL;
+ data_out = FOREACH data_cnt_grouped GENERATE Entropy(data_cnt);
+ store data_out into 'output';
+ */
+ @Multiline private String rawValidInputEntropy;
+
+ @Test
+ public void rawValidInputEntropyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(rawValidInputEntropy);
+
+ writeLinesToFile("input",
+ "0",
+ "38",
+ "0",
+ "62",
+ "38",
+ "32",
+ "96",
+ "38",
+ "96",
+ "0");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > count=c(0, 38, 0, 62, 38, 32, 96, 38, 96, 0)
+ * > library(entropy)
+ * > entropy(count)
+ * [1] 1.846901
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(1.846901);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ /**
+
+ define Entropy datafu.pig.stats.entropy.EmpiricalCountEntropy();
+
+ data_cnt = load 'input' as (val:double);
+ --describe data_cnt;
+ data_cnt_grouped = GROUP data_cnt ALL;
+ data_out = FOREACH data_cnt_grouped GENERATE Entropy(data_cnt);
+ store data_out into 'output';
+ */
+ @Multiline private String rawInvalidTypeInputEntropy;
+
+ @Test
+ public void rawInvalidTypeInputEntropyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(rawInvalidTypeInputEntropy);
+
+ writeLinesToFile("input",
+ "0.0",
+ "38.0",
+ "0.0",
+ "62.0",
+ "38.0",
+ "32.001",
+ "96.002",
+ "38.01",
+ "96.00001",
+ "0.0");
+ try {
+ test.runScript();
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ fail( "Testcase should fail");
+ } catch (Exception ex) {
+ assertTrue(ex.getMessage().indexOf("Expect the type of the input tuple to be of ([int, long]), but instead found double") >= 0);
+ }
+ }
+
+ @Test
+ public void rawInValidInputValueEntropyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(rawValidInputEntropy);
+
+ writeLinesToFile("input",
+ "0",
+ "-38",
+ "0",
+ "62",
+ "38",
+ "32",
+ "96",
+ "38",
+ "96",
+ "0");
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > count=c(0, -38, 0, 62, 38, 32, 96, 38, 96, 0)
+ * > library(entropy)
+ * > entropy(ifelse(count>0,count,0))
+ * [1] 1.693862
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(1.693862);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+
+ }
+
+ /**
+
+ define Entropy datafu.pig.stats.entropy.EmpiricalCountEntropy();
+
+ data_cnt = load 'input' as (f1:chararray, f2:chararray);
+ --describe data_cnt;
+ data_cnt_grouped = GROUP data_cnt ALL;
+ data_out = FOREACH data_cnt_grouped GENERATE Entropy(data_cnt);
+ store data_out into 'output';
+ */
+ @Multiline private String invalidInputSchemaEntropy;
+
+ @Test
+ public void invalidInputSchemaEntropyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(invalidInputSchemaEntropy);
+
+ writeLinesToFile("input",
+ "hadoop 98.94791",
+ "bigdata 38.61010",
+ "hadoop 97.10575",
+ "datafu 32.05370",
+ "bigdata 38.61010",
+ "datafu 32.05370",
+ "datafu 32.05370");
+
+ try {
+ test.runScript();
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ fail( "Testcase should fail");
+ } catch (Exception ex) {
+ assertTrue(ex.getMessage().indexOf("The field schema of the input tuple is null or its size is not 1") >= 0);
+ }
+ }
+
+ /**
+
+ define Entropy datafu.pig.stats.entropy.EmpiricalCountEntropy();
+
+ data = load 'input' as (val:double);
+ --describe data;
+ data_grouped = GROUP data BY val;
+ data_cnt = FOREACH data_grouped GENERATE COUNT(data) AS cnt;
+ data_cnt_grouped = GROUP data_cnt ALL;
+ data_out = FOREACH data_cnt_grouped {
+ data_cnt_ordered = order data_cnt by *;
+ GENERATE Entropy(data_cnt_ordered);
+ }
+ store data_out into 'output';
+ */
+ @Multiline private String accumulatedEntropy;
+
+ @Test
+ public void accumulatedEntropyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(accumulatedEntropy);
+
+ writeLinesToFile("input",
+ "98.94791",
+ "38.61010",
+ "97.10575",
+ "62.28313",
+ "38.61010",
+ "32.05370",
+ "96.10962",
+ "38.61010",
+ "96.10962",
+ "20.41135");
+
+ test.runScript();
+
+ List<Double> expectedOutput = new ArrayList<Double>();
+ //the same output as @test dupValEntropyTest
+ expectedOutput.add(1.834372);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/EntropyTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/EntropyTests.java b/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/EntropyTests.java
new file mode 100644
index 0000000..59fc3c2
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/stats/entropy/EntropyTests.java
@@ -0,0 +1,430 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.stats.entropy;
+
+import static org.testng.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.test.pig.PigTests;
+
+/*
+ * R's entropy library: http://cran.r-project.org/web/packages/entropy/entropy.pdf
+ * used as our test benchmark
+ */
+public class EntropyTests extends AbstractEntropyTests
+{
+ /**
+
+
+ define Entropy datafu.pig.stats.entropy.Entropy();
+
+ data = load 'input' as (val:double);
+ --describe data;
+ data_grouped = GROUP data ALL;
+ data_out = FOREACH data_grouped {
+ data_ordered = ORDER data BY val;
+ GENERATE Entropy(data_ordered);
+ };
+ store data_out into 'output';
+ */
+ @Multiline private String entropy;
+
+ @Test
+ public void uniqValEmpiricalEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(entropy);
+
+ writeLinesToFile("input",
+ "98.94791",
+ "38.61010",
+ "97.10575",
+ "62.28313",
+ "38.83960",
+ "32.05370",
+ "96.10962",
+ "28.72388",
+ "96.65888",
+ "20.41135");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > v=c(98.94791,38.61010,97.10575,62.28313,38.83960,32.05370,96.10962,28.72388,96.65888,20.41135)
+ * > table(v)
+ * v
+ * 20.41135 28.72388 32.0537 38.6101 38.8396 62.28313 96.10962 96.65888 97.10575 98.94791
+ * 1 1 1 1 1 1 1 1 1 1
+ * > count=c(1,1,1,1,1,1,1,1,1,1)
+ * > library(entropy)
+ * > entropy(count)
+ * [1] 2.302585
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(2.302585);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void singleValEmpiricalEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(entropy);
+
+ writeLinesToFile("input",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791",
+ "98.94791");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > v=c(98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791,98.94791)
+ * > table(v)
+ * v
+ * 98.94791
+ * 10
+ * > count=(10)
+ * > library(entropy)
+ * > entropy(count)
+ * [1] 0
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(0.0);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void dupValEmpiricalEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(entropy);
+
+ writeLinesToFile("input",
+ "98.94791",
+ "38.61010",
+ "97.10575",
+ "62.28313",
+ "38.61010",
+ "32.05370",
+ "96.10962",
+ "38.61010",
+ "96.10962",
+ "20.41135");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > v=c(98.94791,38.61010,97.10575,62.28313,38.61010,32.05370,96.10962,38.61010,96.10962,20.41135)
+ * > table(v)
+ * v
+ * 20.41135 32.0537 38.6101 62.28313 96.10962 97.10575 98.94791
+ * 1 1 3 1 2 1 1
+ * > count=c(1,1,3,1,2,1,1)
+ * > library(entropy)
+ * > entropy(count)
+ * [1] 1.834372
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(1.834372);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void emptyInputBagEmpiricalEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(entropy);
+
+ writeLinesToFile("input"
+ );
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > v=c()
+ * > table(v)
+ * < table of extent 0 >
+ * > count=c()
+ * > library(entropy)
+ * > entropy(count)
+ * [1] 0
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void singleElemInputBagEmpiricalEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(entropy);
+
+ writeLinesToFile("input",
+ "98.94791");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > count=c(1)
+ * > library(entropy)
+ * > entropy(count)
+ * [1] 0
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(0.0);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ /**
+
+
+ define Entropy datafu.pig.stats.entropy.Entropy('$type', '$base');
+
+ data = load 'input' as (x:chararray, y:double);
+ --describe data;
+ data_grouped = GROUP data ALL;
+ data_out = FOREACH data_grouped {
+ data_ordered = ORDER data BY x,y;
+ GENERATE Entropy(data_ordered);
+ };
+ store data_out into 'output';
+ */
+ @Multiline private String pairEntropy;
+
+ @Test
+ public void dupPairValEmpiricalEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(pairEntropy, "type=empirical", "base=log");
+
+ writeLinesToFile("input",
+ "hadoop 98.94791",
+ "bigdata 38.61010",
+ "hadoop 97.10575",
+ "datafu 32.05370",
+ "bigdata 38.61010",
+ "datafu 32.05370",
+ "datafu 32.05370",
+ "hadoop 38.61010",
+ "pig 96.10962",
+ "pig 20.41135");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ * > t <- data.table(x=c("hadoop","bigdata","hadoop","datafu","bigdata","datafu","datafu","hadoop","pig","pig"),y=c(98.94791,38.61010,97.10575,32.05370,38.61010,32.05370,32.05370,38.61010,96.10962,20.41135))
+ * > t <- t[order(x,y)]
+ * > count<-c(2,3,1,1,1,1,1)
+ * > library(entropy)
+ * > entropy(count)
+ * [1] 1.834372
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(1.834372);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ /**
+
+
+ define Entropy datafu.pig.stats.entropy.Entropy('$type', '$base');
+
+ data = load 'input' as (val:double);
+ --describe data;
+ data_grouped = GROUP data ALL;
+ data_out = FOREACH data_grouped {
+ data_ordered = ORDER data BY val;
+ GENERATE Entropy(data_ordered);
+ };
+ store data_out into 'output';
+ */
+ @Multiline private String logEntropy;
+
+ @Test
+ public void dupValEmpiricalEntropoyLog2Test() throws Exception
+ {
+ PigTest test = createPigTestFromString(logEntropy, "type=empirical", "base=log2");
+
+ writeLinesToFile("input",
+ "98.94791",
+ "38.61010",
+ "97.10575",
+ "62.28313",
+ "38.61010",
+ "32.05370",
+ "96.10962",
+ "38.61010",
+ "96.10962",
+ "20.41135");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > v=c(98.94791,38.61010,97.10575,62.28313,38.61010,32.05370,96.10962,38.61010,96.10962,20.41135)
+ * > table(v)
+ * v
+ * 20.41135 28.72388 32.0537 38.6101 38.8396 62.28313 96.10962 96.65888 97.10575 98.94791
+ * 1 1 3 1 2 1 1
+ * > count=c(1,1,3,1,2,1,1)
+ * > freqs=count/sum(count)
+ * > library(entropy)
+ * > entropy(count, freqs, c("ML"), c("log2"))
+ * [1] 2.646439
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(2.646439);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ @Test
+ public void dupValEmpiricalEntropoyLog10Test() throws Exception
+ {
+ PigTest test = createPigTestFromString(logEntropy, "type=empirical", "base=log10");
+
+ writeLinesToFile("input",
+ "98.94791",
+ "38.61010",
+ "97.10575",
+ "62.28313",
+ "38.61010",
+ "32.05370",
+ "96.10962",
+ "38.61010",
+ "96.10962",
+ "20.41135");
+
+ test.runScript();
+
+ /* Add expected values, computed using R:
+ *
+ * e.g.
+ *
+ * > v=c(98.94791,38.61010,97.10575,62.28313,38.61010,32.05370,96.10962,38.61010,96.10962,20.41135)
+ * > table(v)
+ * v
+ * 20.41135 28.72388 32.0537 38.6101 38.8396 62.28313 96.10962 96.65888 97.10575 98.94791
+ * 1 1 3 1 2 1 1
+ * > count=c(1,1,3,1,2,1,1)
+ * > freqs=count/sum(count)
+ * > library(entropy)
+ * > entropy(count, freqs, c("ML"), c("log10"))
+ * [1] 0.7966576
+ *
+ */
+ List<Double> expectedOutput = new ArrayList<Double>();
+ expectedOutput.add(0.7966576);
+
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+
+ verifyEqualEntropyOutput(expectedOutput, output, 5);
+ }
+
+ /**
+
+
+ define Entropy datafu.pig.stats.entropy.Entropy();
+
+ data = load 'input' as (val:double);
+ --describe data;
+ data_grouped = GROUP data ALL;
+ data_out = FOREACH data_grouped {
+ GENERATE Entropy(data);
+ };
+ store data_out into 'output';
+ */
+ @Multiline private String noOrderEntropy;
+
+ @Test
+ public void noOrderEmpiricalEntropoyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(noOrderEntropy);
+
+ writeLinesToFile("input",
+ "98.94791",
+ "38.61010",
+ "38.61010",
+ "37.10575",
+ "62.28313",
+ "38.61010",
+ "32.05370",
+ "96.10962",
+ "38.61010",
+ "96.10962",
+ "20.41135");
+
+ try {
+ test.runScript();
+ List<Tuple> output = this.getLinesForAlias(test, "data_out");
+ fail( "Testcase should fail");
+ } catch(Exception ex) {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/text/NLPTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/text/NLPTests.java b/datafu-pig/src/test/java/datafu/test/pig/text/NLPTests.java
new file mode 100644
index 0000000..0504b52
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/text/NLPTests.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.text;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.test.pig.PigTests;
+
+
+public class NLPTests extends PigTests
+{
+ /**
+
+ define SentenceDetect datafu.pig.text.opennlp.SentenceDetect('$DATA_DIR/en-sent.bin');
+
+ data = LOAD 'input' AS (text: chararray);
+
+ dump data;
+
+ data2 = FOREACH data GENERATE SentenceDetect(text) AS sentences;
+
+ dump data2;
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline
+ private String sentenceDetectTest;
+
+ @Test
+ public void sentenceDetectTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(sentenceDetectTest);
+
+ writeLinesToFile("input",
+ "This is a sentence. This is another sentence.",
+ "Yet another sentence. One more just for luck.");
+
+ assertOutput(test, "data2",
+ "({(This is a sentence.),(This is another sentence.)})",
+ "({(Yet another sentence.),(One more just for luck.)})");
+ }
+
+ /**
+
+ define TokenizeME datafu.pig.text.opennlp.TokenizeME('$DATA_DIR/en-token.bin');
+
+ data = LOAD 'input' AS (text: chararray);
+
+ dump data;
+
+ data2 = FOREACH data GENERATE TokenizeME(text) AS tokens;
+
+ dump data2;
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline
+ private String tokenizeMETest;
+
+ @Test
+ public void tokenizeMETest() throws Exception
+ {
+ PigTest test = createPigTestFromString(tokenizeMETest);
+
+ writeLinesToFile("input",
+ "This is a sentence. This is another sentence.",
+ "Yet another sentence. One more just for luck.");
+
+ assertOutput(test, "data2",
+ "({(This),(is),(a),(sentence),(.),(This),(is),(another),(sentence),(.)})",
+ "({(Yet),(another),(sentence),(.),(One),(more),(just),(for),(luck),(.)})");
+ }
+
+ /**
+
+ define TokenizeSimple datafu.pig.text.opennlp.TokenizeSimple();
+
+ data = LOAD 'input' AS (text: chararray);
+
+ dump data;
+
+ data2 = FOREACH data GENERATE TokenizeSimple(text) AS tokens;
+
+ dump data2;
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline
+ private String tokenizeSimpleTest;
+
+ @Test
+ public void tokenizeSimpleTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(tokenizeSimpleTest);
+
+ writeLinesToFile("input",
+ "This is a sentence. This is another sentence.",
+ "Yet another sentence. One more just for luck.");
+
+ assertOutput(test, "data2",
+ "({(This),(is),(a),(sentence),(.),(This),(is),(another),(sentence),(.)})",
+ "({(Yet),(another),(sentence),(.),(One),(more),(just),(for),(luck),(.)})");
+ }
+
+ /**
+
+ define TokenizeWhitespace datafu.pig.text.opennlp.TokenizeWhitespace();
+
+ data = LOAD 'input' AS (text: chararray);
+
+ dump data;
+
+ data2 = FOREACH data GENERATE TokenizeWhitespace(text) AS tokens;
+
+ dump data2;
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline
+ private String tokenizeWhitespaceTest;
+
+ @Test
+ public void tokenizeWhitespaceTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(tokenizeWhitespaceTest);
+
+ writeLinesToFile("input",
+ "This is a sentence. This is another sentence.",
+ "Yet another sentence. One more just for luck.");
+
+ assertOutput(test, "data2",
+ "({(This),(is),(a),(sentence.),(This),(is),(another),(sentence.)})",
+ "({(Yet),(another),(sentence.),(One),(more),(just),(for),(luck.)})");
+ }
+
+ /**
+
+ define TokenizeME datafu.pig.text.opennlp.TokenizeME('$DATA_DIR/en-token.bin');
+ define POSTag datafu.pig.text.opennlp.POSTag('$DATA_DIR/en-pos-maxent.bin');
+
+ data = LOAD 'input' AS (text: chararray);
+
+ dump data;
+
+ data2 = FOREACH data GENERATE TokenizeME(text) AS tokens;
+
+ dump data2;
+
+ data3 = FOREACH data2 GENERATE POSTag(tokens) as tagged;
+
+ dump data3
+
+ STORE data3 INTO 'output';
+ */
+ @Multiline
+ private String POSTagTest;
+
+ @Test
+ public void POSTagTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(POSTagTest);
+
+ writeLinesToFile("input",
+ "This is a sentence. This is another sentence.",
+ "Yet another sentence. One more just for luck.");
+
+ assertOutput(test, "data3",
+ "({(This,DT,0.9649410482478001),(is,VBZ,0.9982592902509803),(a,DT,0.9967282012835504),(sentence,NN,0.9772619256460584),(.,.,0.4391067883074289),(This,DT,0.8346710130761914),(is,VBZ,0.9928885242823617),(another,DT,0.9761159923140399),(sentence,NN,0.9964463493238542),(.,.,0.9856037689871404)})",
+ "({(Yet,RB,0.7638997090011364),(another,DT,0.9657669183153523),(sentence,NN,0.989193114719676),(.,.,0.20091718589945456),(One,CD,0.9229251494813668),(more,JJR,0.9360382000551335),(just,RB,0.8646324491545225),(for,IN,0.9851765355889605),(luck,NN,0.9883408827371651),(.,.,0.9746378518791978)})");
+ }
+}
[10/19] DATAFU-27 Migrate build system to Gradle
Posted by wv...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/urls/UserAgentTest.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/urls/UserAgentTest.java b/datafu-pig/src/test/java/datafu/test/pig/urls/UserAgentTest.java
new file mode 100644
index 0000000..5ebe778
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/urls/UserAgentTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.urls;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.test.pig.PigTests;
+
+public class UserAgentTest extends PigTests
+{
+
+ /**
+
+
+ define UserAgentClassify datafu.pig.urls.UserAgentClassify();
+
+ data = load 'input' as (usr_agent:chararray);
+ data_out = foreach data generate UserAgentClassify(usr_agent) as class;
+ --describe data_out;
+ store data_out into 'output';
+ */
+ @Multiline private String userAgentTest;
+
+ @Test
+ public void userAgentTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(userAgentTest);
+
+ String[] input = {
+ "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
+ "Mozilla/5.0 (compatible; Konqueror/3.5; Linux; X11; de) KHTML/3.5.2 (like Gecko) Kubuntu 6.06 Dapper",
+ "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.2a1pre) Gecko/20110331 Firefox/4.2a1pre Fennec/4.1a1pre",
+ "Opera/9.00 (X11; Linux i686; U; en)",
+ "Wget/1.10.2",
+ "Opera/9.80 (Android; Linux; Opera Mobi/ADR-1012221546; U; pl) Presto/2.7.60 Version/10.5",
+ "Mozilla/5.0 (Linux; U; Android 2.2; en-us; DROID2 Build/VZW) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1"
+ };
+
+ String[] output = {
+ "(mobile)",
+ "(desktop)",
+ "(mobile)",
+ "(desktop)",
+ "(desktop)",
+ "(mobile)",
+ "(mobile)",
+ };
+
+ test.assertOutput("data",input,"data_out",output);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/util/AliasEvalFuncTest.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/util/AliasEvalFuncTest.java b/datafu-pig/src/test/java/datafu/test/pig/util/AliasEvalFuncTest.java
new file mode 100644
index 0000000..3b23d35
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/util/AliasEvalFuncTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.util;
+
+import static org.testng.Assert.*;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.io.IOException;
+
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.pigunit.PigTest;
+import org.apache.pig.data.DataType;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import datafu.test.pig.PigTests;
+import datafu.pig.util.AliasableEvalFunc;
+
+public class AliasEvalFuncTest extends PigTests
+{
+ static class ReportBuilder extends AliasableEvalFunc<DataBag> {
+ static final String ORDERED_ROUTES = "orderedRoutes";
+
+ public DataBag exec(Tuple input) throws IOException {
+ DataBag inputBag = getBag(input, ORDERED_ROUTES);
+ DataBag outputBag = BagFactory.getInstance().newDefaultBag();
+ for(Iterator<Tuple> tupleIter = inputBag.iterator(); tupleIter.hasNext(); ) {
+ outputBag.add(tupleIter.next());
+ }
+ return outputBag;
+ }
+
+ public Schema getOutputSchema(Schema input) {
+ try {
+ Schema bagSchema = input.getField(0).schema;
+ Schema outputSchema = new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
+ .getName()
+ .toLowerCase(), input),
+ bagSchema,
+ DataType.BAG));
+ return outputSchema;
+ } catch (Exception ex) {
+ return null;
+ }
+ }
+ }
+
+ @Test
+ public void getBagTest() throws Exception
+ {
+ ReportBuilder udf = new ReportBuilder();
+ udf.setUDFContextSignature("test");
+ List<Schema.FieldSchema> fieldSchemaList = new ArrayList<Schema.FieldSchema>();
+ fieldSchemaList.add(new Schema.FieldSchema("msisdn", DataType.LONG));
+ fieldSchemaList.add(new Schema.FieldSchema("ts", DataType.INTEGER));
+ fieldSchemaList.add(new Schema.FieldSchema("center_lon", DataType.DOUBLE));
+ fieldSchemaList.add(new Schema.FieldSchema("center_lat", DataType.DOUBLE));
+ Schema schemaTuple = new Schema(fieldSchemaList);
+ Schema schemaBag = new Schema(new Schema.FieldSchema(ReportBuilder.ORDERED_ROUTES, schemaTuple, DataType.BAG));
+ udf.outputSchema(schemaBag);
+
+ Tuple inputTuple = TupleFactory.getInstance().newTuple();
+ DataBag inputBag = BagFactory.getInstance().newDefaultBag();
+ inputBag.add(TupleFactory.getInstance().newTuple(Arrays.asList(71230000000L, 1382351612, 10.697, 20.713)));
+ inputTuple.append(inputBag);
+ DataBag outputBag = udf.exec(inputTuple);
+ Assert.assertEquals(inputBag, outputBag);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/util/AssertTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/util/AssertTests.java b/datafu-pig/src/test/java/datafu/test/pig/util/AssertTests.java
new file mode 100644
index 0000000..1a6a741
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/util/AssertTests.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.util;
+
+import static org.testng.Assert.*;
+
+import java.util.List;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import datafu.test.pig.PigTests;
+
+public class AssertTests extends PigTests
+{
+ /**
+
+
+ define ASRT datafu.pig.util.AssertUDF();
+
+ data = LOAD 'input' AS (val:INT);
+
+ data2 = FILTER data BY ASRT(val,'assertion appears to have failed, doh!');
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline private static String assertWithMessage;
+
+ @Test
+ public void shouldAssertWithMessageOnZero() throws Exception
+ {
+ try
+ {
+ PigTest test = createPigTestFromString(assertWithMessage);
+
+ this.writeLinesToFile("input", "0");
+
+ test.runScript();
+
+ this.getLinesForAlias(test, "data2");
+
+ fail("test should have failed, but it didn't");
+ }
+ catch (Exception e)
+ {
+ }
+ }
+
+ @Test
+ public void shouldNotAssertWithMessageOnOne() throws Exception
+ {
+ PigTest test = createPigTestFromString(assertWithMessage);
+
+ this.writeLinesToFile("input", "1");
+
+ test.runScript();
+
+ List<Tuple> result = this.getLinesForAlias(test, "data2");
+ Assert.assertEquals(result.size(), 1);
+ Assert.assertEquals(result.get(0).size(), 1);
+ Assert.assertEquals(result.get(0).get(0), 1);
+ }
+
+ /**
+
+
+ define ASRT datafu.pig.util.AssertUDF();
+
+ data = LOAD 'input' AS (val:INT);
+
+ data2 = FILTER data BY ASRT(val);
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline private static String assertWithoutMessage;
+
+ @Test
+ public void shouldAssertWithoutMessageOnZero() throws Exception
+ {
+ try
+ {
+ PigTest test = createPigTestFromString(assertWithoutMessage);
+
+ this.writeLinesToFile("input", "0");
+
+ test.runScript();
+
+ this.getLinesForAlias(test, "data2");
+
+ fail("test should have failed, but it didn't");
+ }
+ catch (Exception e)
+ {
+ }
+ }
+
+ @Test
+ public void shouldNotAssertWithoutMessageOnOne() throws Exception
+ {
+ PigTest test = createPigTestFromString(assertWithoutMessage);
+
+ this.writeLinesToFile("input", "1");
+
+ test.runScript();
+
+ List<Tuple> result = this.getLinesForAlias(test, "data2");
+ Assert.assertEquals(result.size(), 1);
+ Assert.assertEquals(result.get(0).size(), 1);
+ Assert.assertEquals(result.get(0).get(0), 1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/util/CoalesceTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/util/CoalesceTests.java b/datafu-pig/src/test/java/datafu/test/pig/util/CoalesceTests.java
new file mode 100644
index 0000000..3baf251
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/util/CoalesceTests.java
@@ -0,0 +1,532 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.util;
+
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.pigunit.PigTest;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.testng.annotations.Test;
+
+import datafu.test.pig.PigTests;
+
+public class CoalesceTests extends PigTests
+{
+ /**
+
+
+ define COALESCE datafu.pig.util.Coalesce();
+
+ data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:INT,val2:INT,val3:INT);
+
+ data2 = FOREACH data GENERATE testcase, COALESCE(val1,val2,val3) as result;
+
+ describe data2;
+
+ data3 = FOREACH data2 GENERATE testcase, result;
+
+ STORE data3 INTO 'output';
+ */
+ @Multiline private static String coalesceIntTest;
+
+ @Test
+ public void coalesceIntTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(coalesceIntTest);
+
+ this.writeLinesToFile("input", "1,1,2,3",
+ "2,,2,3",
+ "3,,,3",
+ "4,,,",
+ "5,1,,3",
+ "6,1,,");
+
+ test.runScript();
+
+ List<Tuple> lines = this.getLinesForAlias(test, "data3");
+
+ Assert.assertEquals(6, lines.size());
+ for (Tuple t : lines)
+ {
+ switch((Integer)t.get(0))
+ {
+ case 1:
+ Assert.assertEquals(1, t.get(1)); break;
+ case 2:
+ Assert.assertEquals(2, t.get(1)); break;
+ case 3:
+ Assert.assertEquals(3, t.get(1)); break;
+ case 4:
+ Assert.assertEquals(null, t.get(1)); break;
+ case 5:
+ Assert.assertEquals(1, t.get(1)); break;
+ case 6:
+ Assert.assertEquals(1, t.get(1)); break;
+ default:
+ Assert.fail("Did not expect: " + t.get(1));
+ }
+ }
+ }
+
+ /**
+
+
+ define COALESCE datafu.pig.util.Coalesce();
+
+ data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:LONG);
+
+ data2 = FOREACH data GENERATE testcase, COALESCE(val1,100L) as result;
+
+ describe data2;
+
+ data3 = FOREACH data2 GENERATE testcase, result;
+
+ data4 = FOREACH data3 GENERATE testcase, result*100 as result;
+
+ STORE data4 INTO 'output';
+ */
+ @Multiline private static String coalesceLongTest;
+
+ @Test
+ public void coalesceLongTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(coalesceLongTest);
+
+ this.writeLinesToFile("input", "1,5",
+ "2,");
+
+ test.runScript();
+
+ List<Tuple> lines = this.getLinesForAlias(test, "data4");
+
+ Assert.assertEquals(2, lines.size());
+ for (Tuple t : lines)
+ {
+ switch((Integer)t.get(0))
+ {
+ case 1:
+ Assert.assertEquals(500L, t.get(1)); break;
+ case 2:
+ Assert.assertEquals(10000L, t.get(1)); break;
+ default:
+ Assert.fail("Did not expect: " + t.get(1));
+ }
+ }
+ }
+
+ /**
+
+
+ define COALESCE datafu.pig.util.Coalesce();
+
+ data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:LONG);
+
+ data2 = FOREACH data GENERATE testcase, COALESCE(val1,100) as result;
+
+ describe data2;
+
+ data3 = FOREACH data2 GENERATE testcase, result;
+
+ data4 = FOREACH data3 GENERATE testcase, result*100 as result;
+
+ STORE data4 INTO 'output';
+ */
+ @Multiline private static String coalesceCastIntToLongTestFails;
+
+ // The first parameter is a long and the fixed value is an int.
+ // They cannot be merged without the lazy option.
+ @Test(expectedExceptions=FrontendException.class)
+ public void coalesceCastIntToLongTestFails() throws Exception
+ {
+ PigTest test = createPigTestFromString(coalesceCastIntToLongTestFails);
+
+ this.writeLinesToFile("input", "1,5",
+ "2,");
+
+ test.runScript();
+
+ List<Tuple> lines = this.getLinesForAlias(test, "data4");
+
+ Assert.assertEquals(2, lines.size());
+ for (Tuple t : lines)
+ {
+ switch((Integer)t.get(0))
+ {
+ case 1:
+ Assert.assertEquals(500L, t.get(1)); break;
+ case 2:
+ Assert.assertEquals(10000L, t.get(1)); break;
+ default:
+ Assert.fail("Did not expect: " + t.get(1));
+ }
+ }
+ }
+
+ /**
+
+
+ define COALESCE datafu.pig.util.Coalesce('lazy');
+
+ data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:LONG);
+
+ data2 = FOREACH data GENERATE testcase, COALESCE(val1,100) as result;
+
+ describe data2;
+
+ data3 = FOREACH data2 GENERATE testcase, result;
+
+ data4 = FOREACH data3 GENERATE testcase, result*100 as result;
+
+ STORE data4 INTO 'output';
+ */
+ @Multiline private static String coalesceIntAndLongTest;
+
+ // The first parameter is a long and the fixed value is an int.
+ // They are merged to a long.
+ @Test
+ public void coalesceCastIntToLongTest1() throws Exception
+ {
+ PigTest test = createPigTestFromString(coalesceIntAndLongTest);
+
+ this.writeLinesToFile("input", "1,5",
+ "2,");
+
+ test.runScript();
+
+ List<Tuple> lines = this.getLinesForAlias(test, "data4");
+
+ Assert.assertEquals(2, lines.size());
+ for (Tuple t : lines)
+ {
+ switch((Integer)t.get(0))
+ {
+ case 1:
+ Assert.assertEquals(500L, t.get(1)); break;
+ case 2:
+ Assert.assertEquals(10000L, t.get(1)); break;
+ default:
+ Assert.fail("Did not expect: " + t.get(1));
+ }
+ }
+ }
+
+ /**
+
+
+ define COALESCE datafu.pig.util.Coalesce('lazy');
+
+ data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:INT);
+
+ data2 = FOREACH data GENERATE testcase, COALESCE(val1,100L) as result;
+
+ describe data2;
+
+ data3 = FOREACH data2 GENERATE testcase, result;
+
+ data4 = FOREACH data3 GENERATE testcase, result*100 as result;
+
+ STORE data4 INTO 'output';
+ */
+ @Multiline private static String coalesceIntAndLongTest2;
+
+ // The first parameter is an int, but the fixed parameter is a long.
+ // They are merged to a long.
+ @Test
+ public void coalesceCastIntToLongTest2() throws Exception
+ {
+ PigTest test = createPigTestFromString(coalesceIntAndLongTest2);
+
+ this.writeLinesToFile("input", "1,5",
+ "2,");
+
+ test.runScript();
+
+ List<Tuple> lines = this.getLinesForAlias(test, "data4");
+
+ Assert.assertEquals(2, lines.size());
+ for (Tuple t : lines)
+ {
+ switch((Integer)t.get(0))
+ {
+ case 1:
+ Assert.assertEquals(500L, t.get(1)); break;
+ case 2:
+ Assert.assertEquals(10000L, t.get(1)); break;
+ default:
+ Assert.fail("Did not expect: " + t.get(1));
+ }
+ }
+ }
+
+ /**
+
+
+ define COALESCE datafu.pig.util.Coalesce('lazy');
+
+ data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:INT);
+
+ data2 = FOREACH data GENERATE testcase, COALESCE(val1,100.0) as result;
+
+ describe data2;
+
+ data3 = FOREACH data2 GENERATE testcase, result;
+
+ data4 = FOREACH data3 GENERATE testcase, result*100 as result;
+
+ STORE data4 INTO 'output';
+ */
+ @Multiline private static String coalesceIntAndDoubleTest;
+
+ // The first parameter is an int, but the fixed parameter is a long.
+ // They are merged to a long.
+ @Test
+ public void coalesceCastIntToDoubleTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(coalesceIntAndDoubleTest);
+
+ this.writeLinesToFile("input", "1,5",
+ "2,");
+
+ test.runScript();
+
+ List<Tuple> lines = this.getLinesForAlias(test, "data4");
+
+ Assert.assertEquals(2, lines.size());
+ for (Tuple t : lines)
+ {
+ switch((Integer)t.get(0))
+ {
+ case 1:
+ Assert.assertEquals(500.0, t.get(1)); break;
+ case 2:
+ Assert.assertEquals(10000.0, t.get(1)); break;
+ default:
+ Assert.fail("Did not expect: " + t.get(1));
+ }
+ }
+ }
+
+ /**
+
+
+ define COALESCE datafu.pig.util.Coalesce();
+
+ data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:LONG);
+
+ data = FOREACH data GENERATE testcase, (val1 IS NOT NULL ? ToDate(val1) : (datetime)null) as val1;
+
+ data2 = FOREACH data GENERATE testcase, COALESCE(val1,ToDate('1970-01-01T00:00:00.000Z')) as result;
+
+ --describe data2;
+
+ data3 = FOREACH data2 GENERATE testcase, result;
+
+ STORE data3 INTO 'output';
+ */
+ @Multiline private static String coalesceCastIntToDatetimeTest;
+
+ @Test
+ public void coalesceCastIntToDatetimeTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(coalesceCastIntToDatetimeTest);
+
+ this.writeLinesToFile("input", "1,1375826183000",
+ "2,");
+
+ test.runScript();
+
+ List<Tuple> lines = this.getLinesForAlias(test, "data3");
+
+ Assert.assertEquals(2, lines.size());
+ for (Tuple t : lines)
+ {
+ Integer testcase = (Integer)t.get(0);
+ Assert.assertNotNull(testcase);
+ switch(testcase)
+ {
+ case 1:
+ Assert.assertEquals("2013-08-06T21:56:23.000Z", ((DateTime)t.get(1)).toDateTime(DateTimeZone.UTC).toString()); break;
+ case 2:
+ Assert.assertEquals("1970-01-01T00:00:00.000Z", t.get(1).toString()); break;
+ default:
+ Assert.fail("Did not expect: " + t.get(1));
+ }
+ }
+ }
+
+ /**
+
+
+ define COALESCE datafu.pig.util.Coalesce('lazy');
+
+ data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:LONG);
+
+ data = FOREACH data GENERATE testcase, (val1 IS NOT NULL ? ToDate(val1) : (datetime)null) as val1;
+
+ data2 = FOREACH data GENERATE testcase, COALESCE(val1,ToDate('1970-01-01T00:00:00.000Z')) as result;
+
+ --describe data2;
+
+ data3 = FOREACH data2 GENERATE testcase, result;
+
+ STORE data3 INTO 'output';
+ */
+ @Multiline private static String coalesceCastIntToDatetimeLazyTest;
+
+ @Test
+ public void coalesceCastIntToDatetimeLazyTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(coalesceCastIntToDatetimeLazyTest);
+
+ this.writeLinesToFile("input", "1,1375826183000",
+ "2,");
+
+ test.runScript();
+
+ List<Tuple> lines = this.getLinesForAlias(test, "data3");
+
+ Assert.assertEquals(2, lines.size());
+ for (Tuple t : lines)
+ {
+ Integer testcase = (Integer)t.get(0);
+ Assert.assertNotNull(testcase);
+ switch(testcase)
+ {
+ case 1:
+ Assert.assertEquals("2013-08-06T21:56:23.000Z", ((DateTime)t.get(1)).toDateTime(DateTimeZone.UTC).toString()); break;
+ case 2:
+ Assert.assertEquals("1970-01-01T00:00:00.000Z", t.get(1).toString()); break;
+ default:
+ Assert.fail("Did not expect: " + t.get(1));
+ }
+ }
+ }
+
+ /**
+
+
+ define COALESCE datafu.pig.util.Coalesce();
+
+ data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:INT,val2:LONG);
+
+ data2 = FOREACH data GENERATE testcase, COALESCE(val1,val2) as result;
+
+ describe data2;
+
+ data3 = FOREACH data2 GENERATE testcase, result;
+
+ STORE data3 INTO 'output';
+ */
+ @Multiline private static String coalesceBagIncompatibleTypeTest;
+
+ @Test(expectedExceptions=FrontendException.class)
+ public void coalesceBagIncompatibleTypeTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(coalesceBagIncompatibleTypeTest);
+
+ this.writeLinesToFile("input", "1,1,2L}");
+
+ test.runScript();
+
+ this.getLinesForAlias(test, "data3");
+ }
+
+ /**
+
+
+ define COALESCE datafu.pig.util.Coalesce('lazy');
+ define EmptyBagToNullFields datafu.pig.bags.EmptyBagToNullFields();
+
+ input1 = LOAD 'input1' using PigStorage(',') AS (val1:INT,val2:INT);
+ input2 = LOAD 'input2' using PigStorage(',') AS (val1:INT,val2:INT);
+ input3 = LOAD 'input3' using PigStorage(',') AS (val1:INT,val2:INT);
+
+ data4 = COGROUP input1 BY val1,
+ input2 BY val1,
+ input3 BY val1;
+
+ dump data4;
+
+ data4 = FOREACH data4 GENERATE
+ FLATTEN(input1),
+ FLATTEN(EmptyBagToNullFields(input2)),
+ FLATTEN(EmptyBagToNullFields(input3));
+
+ dump data4;
+
+ describe data4;
+
+ data5 = FOREACH data4 GENERATE input1::val1 as val1, COALESCE(input2::val2,0L) as val2, COALESCE(input3::val2,0L) as val3;
+
+ --describe data5;
+
+ STORE data5 INTO 'output';
+ */
+ @Multiline private static String leftJoinTest;
+
+ @Test
+ public void leftJoinTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(leftJoinTest);
+
+ this.writeLinesToFile("input1", "1,1",
+ "2,2",
+ "5,5");
+
+ this.writeLinesToFile("input2", "1,10",
+ "3,30",
+ "5,50");
+
+ this.writeLinesToFile("input3", "2,100",
+ "5,500");
+
+ test.runScript();
+
+ List<Tuple> lines = this.getLinesForAlias(test, "data5");
+
+ Assert.assertEquals(3, lines.size());
+ for (Tuple t : lines)
+ {
+ switch((Integer)t.get(0))
+ {
+ case 1:
+ Assert.assertEquals(10L, t.get(1));
+ Assert.assertEquals(0L, t.get(2));
+ break;
+ case 2:
+ Assert.assertEquals(0L, t.get(1));
+ Assert.assertEquals(100L, t.get(2));
+ break;
+ case 5:
+ Assert.assertEquals(50L, t.get(1));
+ Assert.assertEquals(500L, t.get(2));
+ break;
+ default:
+ Assert.fail("Did not expect: " + t.get(0));
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/util/InTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/util/InTests.java b/datafu-pig/src/test/java/datafu/test/pig/util/InTests.java
new file mode 100644
index 0000000..591ee82
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/util/InTests.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.util;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.test.pig.PigTests;
+
+public class InTests extends PigTests
+{
+ /**
+
+
+ define I datafu.pig.util.InUDF();
+
+ data = LOAD 'input' AS (B: bag {T: tuple(v:INT)});
+
+ data2 = FOREACH data {
+ C = FILTER B By I(v, 1,2,3);
+ GENERATE C;
+ }
+
+ describe data2;
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline private static String inIntTest;
+
+ @Test
+ public void inIntTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(inIntTest);
+
+ writeLinesToFile("input",
+ "({(1),(2),(3),(4),(5)})",
+ "({(1),(2)})",
+ "({(4),(5)})");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "({(1),(2),(3)})",
+ "({(1),(2)})",
+ "({})");
+ }
+
+ /**
+
+
+ define I datafu.pig.util.InUDF();
+
+ data = LOAD 'input' AS (B: bag {T: tuple(v:chararray)});
+
+ data2 = FOREACH data {
+ C = FILTER B By I(v, 'will','matt','sam');
+ GENERATE C;
+ }
+
+ describe data2;
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline private static String inStringTest;
+
+ @Test
+ public void inStringTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(inStringTest);
+
+ writeLinesToFile("input",
+ "({(alice),(bob),(will),(matt),(sam)})",
+ "({(will),(matt)})",
+ "({(alice),(bob)})");
+
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "({(will),(matt),(sam)})",
+ "({(will),(matt)})",
+ "({})");
+ }
+
+ /**
+
+
+ define I datafu.pig.util.InUDF();
+
+ data = LOAD 'input' AS (owner:chararray, color:chararray);
+ describe data;
+ data2 = FILTER data BY I(color, 'red','blue');
+ describe data2;
+ STORE data2 INTO 'output';
+ */
+ @Multiline private static String inTopLevelTest;
+
+ @Test
+ public void inTopLevelTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(inTopLevelTest);
+
+ writeLinesToFile("input",
+ "alice\tred",
+ "bob\tblue",
+ "charlie\tgreen",
+ "dave\tred");
+ test.runScript();
+
+ assertOutput(test, "data2",
+ "(alice,red)",
+ "(bob,blue)",
+ "(dave,red)");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/util/IntBoolConversionPigTests.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/util/IntBoolConversionPigTests.java b/datafu-pig/src/test/java/datafu/test/pig/util/IntBoolConversionPigTests.java
new file mode 100644
index 0000000..465e8b2
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/util/IntBoolConversionPigTests.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.util;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.annotations.Test;
+
+import datafu.test.pig.PigTests;
+
+public class IntBoolConversionPigTests extends PigTests
+{
+ /**
+
+
+ define IntToBool datafu.pig.util.IntToBool();
+
+ data = LOAD 'input' AS (val:INT);
+
+ data2 = FOREACH data GENERATE IntToBool(val);
+
+ STORE data2 INTO 'output';
+ */
+ @Multiline private static String intToBoolTest;
+
+ @Test
+ public void intToBoolTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(intToBoolTest);
+
+ String[] input = {
+ "", // null
+ "0",
+ "1"
+ };
+
+ String[] output = {
+ "(false)",
+ "(false)",
+ "(true)"
+ };
+
+ test.assertOutput("data",input,"data2",output);
+ }
+
+ /**
+
+
+ define IntToBool datafu.pig.util.IntToBool();
+ define BoolToInt datafu.pig.util.BoolToInt();
+
+ data = LOAD 'input' AS (val:INT);
+
+ data2 = FOREACH data GENERATE IntToBool(val) as val;
+ data3 = FOREACH data2 GENERATE BoolToInt(val) as val;
+
+ STORE data3 INTO 'output';
+ */
+ @Multiline private static String intToBoolToIntTest;
+
+ @Test
+ public void intToBoolToIntTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(intToBoolToIntTest);
+
+ String[] input = {
+ "", // null
+ "0",
+ "1",
+ "2",
+ "-1",
+ "-2",
+ "0",
+ ""
+ };
+
+ String[] output = {
+ "(0)",
+ "(0)",
+ "(1)",
+ "(1)",
+ "(1)",
+ "(1)",
+ "(0)",
+ "(0)"
+ };
+
+ test.assertOutput("data",input,"data3",output);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/test/java/datafu/test/pig/util/TransposeTest.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/test/java/datafu/test/pig/util/TransposeTest.java b/datafu-pig/src/test/java/datafu/test/pig/util/TransposeTest.java
new file mode 100644
index 0000000..3d7654a
--- /dev/null
+++ b/datafu-pig/src/test/java/datafu/test/pig/util/TransposeTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.test.pig.util;
+
+import java.util.List;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.pigunit.PigTest;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import datafu.test.pig.PigTests;
+
+public class TransposeTest extends PigTests
+{
+ /**
+
+
+ define Transpose datafu.pig.util.TransposeTupleToBag();
+
+ data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:INT,val2:INT,val3:INT);
+
+ data2 = FOREACH data GENERATE testcase, Transpose(val1 .. val3) as transposed;
+
+ describe data2;
+
+ data3 = FOREACH data2 GENERATE testcase, transposed;
+
+ STORE data3 INTO 'output';
+ */
+ @Multiline private static String transposeTest;
+
+ @Test
+ public void transposeTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(transposeTest);
+ writeLinesToFile("input", "1,10,11,12",
+ "2,20,21,22");
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data3");
+ for (Tuple tuple : output) {
+ int testCase = (Integer)tuple.get(0);
+ DataBag bag = (DataBag)tuple.get(1);
+ Assert.assertEquals(bag.size(), 3);
+ int i=0;
+ for (Tuple t : bag) {
+ String expectedKey = String.format("val%d",i+1);
+ Assert.assertEquals((String)t.get(0), expectedKey);
+ int actualValue = (Integer)t.get(1);
+ Assert.assertEquals(actualValue, testCase*10+i);
+ i++;
+ }
+ }
+ }
+
+ /**
+
+
+ define Transpose datafu.pig.util.TransposeTupleToBag();
+
+ data = LOAD 'input' using PigStorage(',') AS (testcase:INT,val1:INT,val2:INT,val3:DOUBLE);
+
+ data2 = FOREACH data GENERATE testcase, Transpose(val1 .. val3) as transposed;
+
+ describe data2;
+
+ data3 = FOREACH data2 GENERATE testcase, transposed;
+
+ STORE data3 INTO 'output';
+ */
+ @Multiline private static String transposeBadTypeTest;
+
+ @Test(expectedExceptions={org.apache.pig.impl.logicalLayer.FrontendException.class})
+ public void transposeBadTypeTest() throws Exception
+ {
+ PigTest test = createPigTestFromString(transposeBadTypeTest);
+ writeLinesToFile("input", "1,10,11,12.0",
+ "2,20,21,22.0");
+ test.runScript();
+
+ List<Tuple> output = getLinesForAlias(test, "data3");
+ for (Tuple tuple : output) {
+ int testCase = (Integer)tuple.get(0);
+ DataBag bag = (DataBag)tuple.get(1);
+ Assert.assertEquals(bag.size(), 3);
+ int i=0;
+ for (Tuple t : bag) {
+ String expectedKey = String.format("val%d",i+1);
+ Assert.assertEquals((String)t.get(0), expectedKey);
+ int actualValue = (Integer)t.get(1);
+ Assert.assertEquals(actualValue, testCase*10+i);
+ i++;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/gradle.properties
----------------------------------------------------------------------
diff --git a/gradle.properties b/gradle.properties
new file mode 100644
index 0000000..98e86a2
--- /dev/null
+++ b/gradle.properties
@@ -0,0 +1,2 @@
+group=org.apache.datafu
+version=1.2.1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/gradle/buildscript.gradle
----------------------------------------------------------------------
diff --git a/gradle/buildscript.gradle b/gradle/buildscript.gradle
new file mode 100644
index 0000000..225e0a8
--- /dev/null
+++ b/gradle/buildscript.gradle
@@ -0,0 +1,12 @@
+repositories {
+ repositories {
+ // For license plugin.
+ maven {
+ url 'http://dl.bintray.com/content/netflixoss/external-gradle-plugins/'
+ }
+ }
+}
+
+dependencies {
+ classpath 'nl.javadude.gradle.plugins:license-gradle-plugin:0.6.1'
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/gradle/dependency-versions.gradle
----------------------------------------------------------------------
diff --git a/gradle/dependency-versions.gradle b/gradle/dependency-versions.gradle
new file mode 100644
index 0000000..29d3e36
--- /dev/null
+++ b/gradle/dependency-versions.gradle
@@ -0,0 +1,20 @@
+ext {
+ antlrVersion="3.2"
+ avroVersion="1.5.3"
+ streamVersion="2.5.0"
+ commonsMathVersion="2.2"
+ commonsIoVersion="1.4"
+ fastutilVersion="6.5.7"
+ guavaVersion="11.0"
+ hadoopVersion="0.20.2"
+ jodaTimeVersion="1.6"
+ log4jVersion="1.2.14"
+ mavenVersion="2.1.3"
+ jlineVersion="0.9.94"
+ pigVersion="0.11.1"
+ testngVersion="6.2"
+ toolsVersion="1.4.2"
+ wagonHttpVersion="1.0-beta-2"
+ openNlpVersion="1.5.3"
+ openNlpMaxEntVersion="3.0.3"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/gradle/wrapper/gradle-wrapper.jar
----------------------------------------------------------------------
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
new file mode 100644
index 0000000..a7634b0
Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/gradle/wrapper/gradle-wrapper.properties
----------------------------------------------------------------------
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
new file mode 100644
index 0000000..610282a
--- /dev/null
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -0,0 +1,6 @@
+#Thu Jul 11 22:18:11 PDT 2013
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=http\://services.gradle.org/distributions/gradle-1.6-bin.zip
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/gradlew
----------------------------------------------------------------------
diff --git a/gradlew b/gradlew
new file mode 100755
index 0000000..16bbbbf
--- /dev/null
+++ b/gradlew
@@ -0,0 +1,164 @@
+#!/usr/bin/env bash
+
+##############################################################################
+##
+## Gradle start up script for UN*X
+##
+##############################################################################
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS="-XX:MaxPermSize=512m"
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn ( ) {
+ echo "$*"
+}
+
+die ( ) {
+ echo
+ echo "$*"
+ echo
+ exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+case "`uname`" in
+ CYGWIN* )
+ cygwin=true
+ ;;
+ Darwin* )
+ darwin=true
+ ;;
+ MINGW* )
+ msys=true
+ ;;
+esac
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched.
+if $cygwin ; then
+ [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
+fi
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "$PRG"`"/$link"
+ fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >&-
+APP_HOME="`pwd -P`"
+cd "$SAVED" >&-
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ JAVACMD="$JAVA_HOME/jre/sh/java"
+ else
+ JAVACMD="$JAVA_HOME/bin/java"
+ fi
+ if [ ! -x "$JAVACMD" ] ; then
+ die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+ fi
+else
+ JAVACMD="java"
+ which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
+ MAX_FD_LIMIT=`ulimit -H -n`
+ if [ $? -eq 0 ] ; then
+ if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+ MAX_FD="$MAX_FD_LIMIT"
+ fi
+ ulimit -n $MAX_FD
+ if [ $? -ne 0 ] ; then
+ warn "Could not set maximum file descriptor limit: $MAX_FD"
+ fi
+ else
+ warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+ fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+ GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin ; then
+ APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+ CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+
+ # We build the pattern for arguments to be converted via cygpath
+ ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+ SEP=""
+ for dir in $ROOTDIRSRAW ; do
+ ROOTDIRS="$ROOTDIRS$SEP$dir"
+ SEP="|"
+ done
+ OURCYGPATTERN="(^($ROOTDIRS))"
+ # Add a user-defined pattern to the cygpath arguments
+ if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+ OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+ fi
+ # Now convert the arguments - kludge to limit ourselves to /bin/sh
+ i=0
+ for arg in "$@" ; do
+ CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+ CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
+
+ if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
+ eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+ else
+ eval `echo args$i`="\"$arg\""
+ fi
+ i=$((i+1))
+ done
+ case $i in
+ (0) set -- ;;
+ (1) set -- "$args0" ;;
+ (2) set -- "$args0" "$args1" ;;
+ (3) set -- "$args0" "$args1" "$args2" ;;
+ (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+ (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+ (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+ (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+ (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+ (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+ esac
+fi
+
+# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
+function splitJvmOpts() {
+ JVM_OPTS=("$@")
+}
+eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
+JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+
+exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/plugin/java/org/adrianwalker/multilinestring/EcjMultilineProcessor.java
----------------------------------------------------------------------
diff --git a/plugin/java/org/adrianwalker/multilinestring/EcjMultilineProcessor.java b/plugin/java/org/adrianwalker/multilinestring/EcjMultilineProcessor.java
deleted file mode 100644
index 7dbfe9a..0000000
--- a/plugin/java/org/adrianwalker/multilinestring/EcjMultilineProcessor.java
+++ /dev/null
@@ -1,54 +0,0 @@
-// Code from https://github.com/benelog/multiline.git
-// Based on Adrian Walker's blog post: http://www.adrianwalker.org/2011/12/java-multiline-string.html
-
-package org.adrianwalker.multilinestring;
-
-import java.util.Set;
-
-import javax.annotation.processing.AbstractProcessor;
-import javax.annotation.processing.ProcessingEnvironment;
-import javax.annotation.processing.RoundEnvironment;
-import javax.annotation.processing.SupportedAnnotationTypes;
-import javax.annotation.processing.SupportedSourceVersion;
-import javax.lang.model.SourceVersion;
-import javax.lang.model.element.Element;
-import javax.lang.model.element.TypeElement;
-import javax.lang.model.util.Elements;
-
-import org.eclipse.jdt.internal.compiler.apt.model.VariableElementImpl;
-import org.eclipse.jdt.internal.compiler.ast.FieldDeclaration;
-import org.eclipse.jdt.internal.compiler.ast.StringLiteral;
-import org.eclipse.jdt.internal.compiler.lookup.FieldBinding;
-
-import java.lang.reflect.Constructor;
-
-@SupportedAnnotationTypes({"org.adrianwalker.multilinestring.Multiline"})
-@SupportedSourceVersion(SourceVersion.RELEASE_6)
-public final class EcjMultilineProcessor extends AbstractProcessor {
-
- private Elements elementUtils;
-
- @Override
- public void init(final ProcessingEnvironment procEnv) {
- super.init(procEnv);
- this.elementUtils = procEnv.getElementUtils();
- }
-
- @Override
- public boolean process(final Set<? extends TypeElement> annotations, final RoundEnvironment roundEnv) {
- Set<? extends Element> fields = roundEnv.getElementsAnnotatedWith(Multiline.class);
-
- for (Element field : fields) {
- String docComment = elementUtils.getDocComment(field);
-
- if (null != docComment) {
- VariableElementImpl fieldElem = (VariableElementImpl) field;
- FieldBinding biding = (FieldBinding) fieldElem._binding;
- FieldDeclaration decl = biding.sourceField();
- StringLiteral string = new StringLiteral(docComment.toCharArray(), decl.sourceStart, decl.sourceEnd, decl.sourceStart);
- decl.initialization = string;
- }
- }
- return true;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/plugin/java/org/adrianwalker/multilinestring/JavacMultilineProcessor.java
----------------------------------------------------------------------
diff --git a/plugin/java/org/adrianwalker/multilinestring/JavacMultilineProcessor.java b/plugin/java/org/adrianwalker/multilinestring/JavacMultilineProcessor.java
deleted file mode 100644
index 39aa24e..0000000
--- a/plugin/java/org/adrianwalker/multilinestring/JavacMultilineProcessor.java
+++ /dev/null
@@ -1,49 +0,0 @@
-// Code from https://github.com/benelog/multiline.git
-// Based on Adrian Walker's blog post: http://www.adrianwalker.org/2011/12/java-multiline-string.html
-
-package org.adrianwalker.multilinestring;
-
-import java.util.Set;
-
-import javax.annotation.processing.AbstractProcessor;
-import javax.annotation.processing.ProcessingEnvironment;
-import javax.annotation.processing.RoundEnvironment;
-import javax.annotation.processing.SupportedAnnotationTypes;
-import javax.annotation.processing.SupportedSourceVersion;
-import javax.lang.model.SourceVersion;
-import javax.lang.model.element.Element;
-import javax.lang.model.element.TypeElement;
-
-import com.sun.tools.javac.model.JavacElements;
-import com.sun.tools.javac.processing.JavacProcessingEnvironment;
-import com.sun.tools.javac.tree.JCTree.JCVariableDecl;
-import com.sun.tools.javac.tree.TreeMaker;
-
-@SupportedAnnotationTypes({"org.adrianwalker.multilinestring.Multiline"})
-@SupportedSourceVersion(SourceVersion.RELEASE_6)
-public final class JavacMultilineProcessor extends AbstractProcessor {
-
- private JavacElements elementUtils;
- private TreeMaker maker;
-
- @Override
- public void init(final ProcessingEnvironment procEnv) {
- super.init(procEnv);
- JavacProcessingEnvironment javacProcessingEnv = (JavacProcessingEnvironment) procEnv;
- this.elementUtils = javacProcessingEnv.getElementUtils();
- this.maker = TreeMaker.instance(javacProcessingEnv.getContext());
- }
-
- @Override
- public boolean process(final Set<? extends TypeElement> annotations, final RoundEnvironment roundEnv) {
- Set<? extends Element> fields = roundEnv.getElementsAnnotatedWith(Multiline.class);
- for (Element field : fields) {
- String docComment = elementUtils.getDocComment(field);
- if (null != docComment) {
- JCVariableDecl fieldNode = (JCVariableDecl) elementUtils.getTree(field);
- fieldNode.init = maker.Literal(docComment);
- }
- }
- return true;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/plugin/java/org/adrianwalker/multilinestring/Multiline.java
----------------------------------------------------------------------
diff --git a/plugin/java/org/adrianwalker/multilinestring/Multiline.java b/plugin/java/org/adrianwalker/multilinestring/Multiline.java
deleted file mode 100644
index c29d3ef..0000000
--- a/plugin/java/org/adrianwalker/multilinestring/Multiline.java
+++ /dev/null
@@ -1,14 +0,0 @@
-// Code from https://github.com/benelog/multiline.git
-// Based on Adrian Walker's blog post: http://www.adrianwalker.org/2011/12/java-multiline-string.html
-
-package org.adrianwalker.multilinestring;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Target({ElementType.FIELD,ElementType.LOCAL_VARIABLE})
-@Retention(RetentionPolicy.SOURCE)
-public @interface Multiline {
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/plugin/java/org/adrianwalker/multilinestring/MultilineProcessor.java
----------------------------------------------------------------------
diff --git a/plugin/java/org/adrianwalker/multilinestring/MultilineProcessor.java b/plugin/java/org/adrianwalker/multilinestring/MultilineProcessor.java
deleted file mode 100644
index 9abdba5..0000000
--- a/plugin/java/org/adrianwalker/multilinestring/MultilineProcessor.java
+++ /dev/null
@@ -1,41 +0,0 @@
-// Code from https://github.com/benelog/multiline.git
-// Based on Adrian Walker's blog post: http://www.adrianwalker.org/2011/12/java-multiline-string.html
-
-package org.adrianwalker.multilinestring;
-
-import java.util.Set;
-
-import javax.annotation.processing.AbstractProcessor;
-import javax.annotation.processing.ProcessingEnvironment;
-import javax.annotation.processing.Processor;
-import javax.annotation.processing.RoundEnvironment;
-import javax.annotation.processing.SupportedAnnotationTypes;
-import javax.annotation.processing.SupportedSourceVersion;
-import javax.lang.model.SourceVersion;
-import javax.lang.model.element.TypeElement;
-
-@SupportedAnnotationTypes({"org.adrianwalker.multilinestring.Multiline"})
-@SupportedSourceVersion(SourceVersion.RELEASE_6)
-public final class MultilineProcessor extends AbstractProcessor {
- private Processor delegator = null;
-
- @Override
- public void init(final ProcessingEnvironment procEnv) {
- super.init(procEnv);
- String envClassName = procEnv.getClass().getName();
- if (envClassName.contains("com.sun.tools")) {
- delegator = new JavacMultilineProcessor();
- } else {
- delegator = new EcjMultilineProcessor();
- }
- delegator.init(procEnv);
- }
-
- @Override
- public boolean process(final Set<? extends TypeElement> annotations, final RoundEnvironment roundEnv) {
- if (delegator == null ) {
- return true;
- }
- return delegator.process(annotations, roundEnv);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/settings.gradle
----------------------------------------------------------------------
diff --git a/settings.gradle b/settings.gradle
new file mode 100644
index 0000000..62b1899
--- /dev/null
+++ b/settings.gradle
@@ -0,0 +1 @@
+include "build-plugin","datafu-pig"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/bags/AppendToBag.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/bags/AppendToBag.java b/src/java/datafu/pig/bags/AppendToBag.java
deleted file mode 100644
index 55c9e76..0000000
--- a/src/java/datafu/pig/bags/AppendToBag.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.bags;
-
-import java.io.IOException;
-
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-import datafu.pig.util.SimpleEvalFunc;
-
-/**
- * Appends a tuple to a bag.
- * <p>
- * Example:
- * <pre>
- * {@code
- * define AppendToBag datafu.pig.bags.AppendToBag();
- *
- * -- input:
- * -- ({(1),(2),(3)},(4))
- * -- ({(10),(20),(30),(40),(50)},(60))
- * input = LOAD 'input' AS (B: bag{T: tuple(v:INT)}, T: tuple(v:INT));
-
- * -- output:
- * -- ({(1),(2),(3),(4)})
- * -- ({(10),(20),(30),(40),(50),(60)})
- * output = FOREACH input GENERATE AppendToBag(B,T) as B;
- * }
- * </pre>
- */
-public class AppendToBag extends SimpleEvalFunc<DataBag>
-{
- public DataBag call(DataBag inputBag, Tuple t) throws IOException
- {
- inputBag.add(t);
- return inputBag;
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try {
- return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
- input.getField(0).schema, DataType.BAG));
- }
- catch (FrontendException e) {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/bags/BagConcat.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/bags/BagConcat.java b/src/java/datafu/pig/bags/BagConcat.java
deleted file mode 100644
index 290f44b..0000000
--- a/src/java/datafu/pig/bags/BagConcat.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.bags;
-
-import java.io.IOException;
-
-import org.apache.pig.EvalFunc;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
-
-/**
- * Unions all input bags to produce a single bag containing all tuples.
- * <p>
- * This UDF accepts two forms of input:
- * <ol>
- * <li>a tuple of 2 or more elements where each element is a bag with the same schema</li>
- * <li>a single bag where each element of that bag is a bag and all of these bags have the same schema</li>
- * </ol>
- * <p>
- * Example 1:
- * <pre>
- * {@code
- * define BagConcat datafu.pig.bags.BagConcat();
- * -- This example illustrates the use on a tuple of bags
- *
- * -- input:
- * -- ({(1),(2),(3)},{(3),(4),(5)})
- * -- ({(20),(25)},{(40),(50)})
- * input = LOAD 'input' AS (A: bag{T: tuple(v:INT)}, B: bag{T: tuple(v:INT)});
- *
- * -- output:
- * -- ({(1),(2),(3),(3),(4),(5)})
- * -- ({(20),(25),(40),(50)})
- * output = FOREACH input GENERATE BagConcat(A,B);
- * }
- * </pre>
- * <p>
- * Example 2:
- * <pre>
- * {@code
- * define BagConcat datafu.pig.bags.BagConcat();
- * -- This example illustrates the use on a bag of bags
- *
- * -- input:
- * -- ({({(1),(2),(3)}),({(3),(4),(5)})})
- * -- ({({(20),(25)}),({(40),(50)})})
- * input = LOAD 'input' AS (A: bag{T: tuple(bag{T2: tuple(v:INT)})});
- *
- * -- output:
- * -- ({(1),(2),(3),(3),(4),(5)})
- * -- ({(20),(25),(40),(50)})
- * output = FOREACH input GENERATE BagConcat(A);
- * }
- * </pre>
- *
- * @author wvaughan
- *
- */
-public class BagConcat extends EvalFunc<DataBag>
-{
-
- @Override
- public DataBag exec(Tuple input) throws IOException
- {
- DataBag output = BagFactory.getInstance().newDefaultBag();
- if (input.size() > 1) {
- // tuple of bags
- for (int i=0; i < input.size(); i++) {
- Object o = input.get(i);
- if (!(o instanceof DataBag)) {
- throw new RuntimeException("Expected a TUPLE of BAGs as input");
- }
-
- DataBag inputBag = (DataBag) o;
- for (Tuple elem : inputBag) {
- output.add(elem);
- }
- }
- } else {
- // bag of bags
- DataBag outerBag = (DataBag)input.get(0);
- for (Tuple outerTuple : outerBag) {
- DataBag innerBag = (DataBag)outerTuple.get(0);
- for (Tuple innerTuple : innerBag) {
- output.add(innerTuple);
- }
- }
- }
- return output;
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try {
- Schema outputBagTupleSchema = null;
-
- if (input.size() == 0) {
- throw new RuntimeException("Expected input tuple to contain fields. Got none.");
- }
- // determine whether the input is a tuple of bags or a bag of bags
- if (input.size() != 1) {
- // tuple of bags
-
- // verify that each element in the input is a bag
- for (FieldSchema fieldSchema : input.getFields()) {
- if (fieldSchema.type != DataType.BAG) {
- throwBadTypeError("Expected a TUPLE of BAGs as input. Got instead: %s in schema: %s", fieldSchema.type, input);
- }
- if (fieldSchema.schema == null) {
- throwBadSchemaError(fieldSchema.alias, input);
- }
- }
-
- outputBagTupleSchema = input.getField(0).schema;
- } else {
- // bag of bags
-
- // should only be a single element in the input and it should be a bag
- FieldSchema fieldSchema = input.getField(0);
- if (fieldSchema.type != DataType.BAG) {
- throwBadTypeError("Expected a BAG of BAGs as input. Got instead: %s in schema: %s", fieldSchema.type, input);
- }
-
- // take the tuple schema from this outer bag
- Schema outerBagTuple = input.getField(0).schema;
- // verify that this tuple contains only a bag for each element
- Schema outerBagSchema = outerBagTuple.getField(0).schema;
- if (outerBagSchema.size() != 1) {
- throw new RuntimeException(String.format("Expected outer bag to have only a single field. Got instead: %s",
- outerBagSchema.prettyPrint()));
- }
- FieldSchema outerBagFieldSchema = outerBagSchema.getField(0);
- if (outerBagFieldSchema.type != DataType.BAG) {
- throwBadTypeError("Expected a BAG of BAGs as input. Got instead: %s", outerBagFieldSchema.type, outerBagTuple);
- }
-
- // take the schema of the inner tuple as the schema for the tuple in the output bag
- FieldSchema innerTupleSchema = outerBagSchema.getField(0);
- if (innerTupleSchema.schema == null) {
- throwBadSchemaError(innerTupleSchema.alias, outerBagSchema);
- }
- outputBagTupleSchema = innerTupleSchema.schema;
- }
-
- // return the correct schema
- return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
- outputBagTupleSchema, DataType.BAG));
- }
- catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- private void throwBadSchemaError(String alias, Schema schema) throws FrontendException
- {
- throw new FrontendException(String.format("Expected non-null schema for all bags. Got null on field %s, in: %s",
- alias, schema.prettyPrint()));
- }
-
- private void throwBadTypeError(String expectedMessage, byte actualType, Schema schema) throws FrontendException
- {
- throw new FrontendException(String.format(expectedMessage, DataType.findTypeName(actualType), schema.prettyPrint()));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/bags/BagGroup.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/bags/BagGroup.java b/src/java/datafu/pig/bags/BagGroup.java
deleted file mode 100644
index 7b6c2e2..0000000
--- a/src/java/datafu/pig/bags/BagGroup.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.bags;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
-
-import datafu.pig.util.AliasableEvalFunc;
-
-/**
- * Performs an in-memory group operation on a bag. The first argument is the bag.
- * The second argument is a projection of that bag to the group keys.
- *
- * <p>
- * Example:
- * <code>
- * define BagGroup datafu.pig.bags.BagGroup();
- *
- * data = LOAD 'input' AS (input_bag: bag {T: tuple(k: int, v: chararray)});
- * -- ({(1,A),(1,B),(2,A),(2,B),(2,C),(3,A)})
- *
- * data2 = FOREACH data GENERATE BagGroup(input_bag, input_bag.(k)) as grouped;
- * -- data2: {grouped: {(group: int,input_bag: {T: (k: int,v: chararray)})}}
- * -- ({(1,{(1,A),(1,B)}),(2,{(2,A),(2,B),(2,C)}),(3,{(3,A)})})
- * </code>
- * </p>
- *
- * @author wvaughan
- *
- */
-public class BagGroup extends AliasableEvalFunc<DataBag>
-{
- private final String FIELD_NAMES_PROPERTY = "FIELD_NAMES";
- private List<String> fieldNames;
-
- @Override
- public Schema getOutputSchema(Schema input)
- {
- try {
- if (input.size() != 2) {
- throw new RuntimeException(String.format("Expected input of format (BAG, PROJECTED_BAG...). Got %d field.", input.size()));
- }
- // Expect the first field to be a bag
- FieldSchema bagFieldSchema = input.getField(0);
- if (bagFieldSchema.type != DataType.BAG) {
- throw new RuntimeException(String.format("Expected input of format (BAG, PROJECTED_BAG...). Got %s as first field.", DataType.findTypeName(bagFieldSchema.type)));
- }
- // Expect the second fields to be a projection of the bag
- FieldSchema projectedBagFieldSchema = input.getField(1);
- if (projectedBagFieldSchema.type != DataType.BAG) {
- throw new RuntimeException(String.format("Expected input of format (BAG, PROJECTED_BAG...). Got %s as second field.", DataType.findTypeName(projectedBagFieldSchema.type)));
- }
-
- String bagName = bagFieldSchema.alias;
- // handle named tuples
- if (bagFieldSchema.schema.size() == 1) {
- FieldSchema bagTupleFieldSchema = bagFieldSchema.schema.getField(0);
- if (bagTupleFieldSchema.type == DataType.TUPLE && bagTupleFieldSchema.alias != null) {
- bagName = getPrefixedAliasName(bagName, bagTupleFieldSchema.alias);
- }
- }
- if (projectedBagFieldSchema.schema.size() == 1) {
- FieldSchema projectedBagTupleFieldSchema = projectedBagFieldSchema.schema.getField(0);
- if (projectedBagTupleFieldSchema.type == DataType.TUPLE && projectedBagTupleFieldSchema.schema != null) {
- projectedBagFieldSchema = projectedBagTupleFieldSchema;
- }
- }
-
- // create the output schema for the 'group'
- // store the field names for the group keys
- Schema groupTupleSchema = new Schema();
- fieldNames = new ArrayList<String>(projectedBagFieldSchema.schema.size());
- for (int i=0; i<projectedBagFieldSchema.schema.size(); i++) {
- FieldSchema fieldSchema = projectedBagFieldSchema.schema.getField(i);
- String fieldName = fieldSchema.alias;
- fieldNames.add(getPrefixedAliasName(bagName, fieldName));
- groupTupleSchema.add(new FieldSchema(fieldSchema.alias, fieldSchema.type));
- }
- getInstanceProperties().put(FIELD_NAMES_PROPERTY, fieldNames);
-
- Schema outputTupleSchema = new Schema();
- if (projectedBagFieldSchema.schema.size() > 1) {
- // multiple group keys
- outputTupleSchema.add(new FieldSchema("group", groupTupleSchema, DataType.TUPLE));
- } else {
- // single group key
- outputTupleSchema.add(new FieldSchema("group", groupTupleSchema.getField(0).type));
- }
- outputTupleSchema.add(bagFieldSchema);
-
- return new Schema(new Schema.FieldSchema(
- getSchemaName(this.getClass().getName().toLowerCase(), input),
- outputTupleSchema,
- DataType.BAG));
- } catch (FrontendException e) {
- throw new RuntimeException(e);
- }
- }
-
- Map<Tuple, List<Tuple>> groups = new HashMap<Tuple, List<Tuple>>();
- TupleFactory tupleFactory = TupleFactory.getInstance();
- BagFactory bagFactory = BagFactory.getInstance();
-
- @SuppressWarnings("unchecked")
- @Override
- public DataBag exec(Tuple input) throws IOException
- {
- fieldNames = (List<String>)getInstanceProperties().get(FIELD_NAMES_PROPERTY);
-
- DataBag inputBag = (DataBag)input.get(0);
-
- for (Tuple tuple : inputBag) {
- Tuple key = extractKey(tuple);
- addGroup(key, tuple);
- }
-
- DataBag outputBag = bagFactory.newDefaultBag();
- for (Tuple key : groups.keySet()) {
- Tuple outputTuple = tupleFactory.newTuple();
- if (fieldNames.size() > 1) {
- outputTuple.append(key);
- } else {
- outputTuple.append(key.get(0));
- }
- DataBag groupBag = bagFactory.newDefaultBag();
- for (Tuple groupedTuple : groups.get(key)) {
- groupBag.add(groupedTuple);
- }
- outputTuple.append(groupBag);
- outputBag.add(outputTuple);
- }
-
- return outputBag;
- }
-
- private Tuple extractKey(Tuple tuple) throws ExecException {
- Tuple key = tupleFactory.newTuple();
- for (String field : fieldNames) {
- key.append(getObject(tuple, field));
- }
- return key;
- }
-
- private void addGroup(Tuple key, Tuple value) {
- if (!groups.containsKey(key)) {
- groups.put(key, new LinkedList<Tuple>());
- }
- groups.get(key).add(value);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/bags/BagLeftOuterJoin.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/bags/BagLeftOuterJoin.java b/src/java/datafu/pig/bags/BagLeftOuterJoin.java
deleted file mode 100644
index ba6bc11..0000000
--- a/src/java/datafu/pig/bags/BagLeftOuterJoin.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.bags;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
-
-import datafu.pig.util.AliasableEvalFunc;
-
-/**
- * Performs an in-memory left outer join across multiple bags.
- *
- * <p>
- * The format for invocation is BagLeftOuterJoin(bag, 'key',....).
- * This UDF expects that all bags are non-null and that there is a corresponding key for each bag.
- * The <em>key</em> that is expected is the alias of the key inside of the preceding bag.
- * </p>
- *
- * <p>
- * Example:
- * <code>
- * define BagLeftOuterJoin datafu.pig.bags.BagLeftOuterJoin();
- *
- * -- describe data:
- * -- data: {bag1: {(key1: chararray,value1: chararray)},bag2: {(key2: chararray,value2: int)}}
- *
- * bag_joined = FOREACH data GENERATE BagLeftOuterJoin(bag1, 'key1', bag2, 'key2') as joined;
- *
- * -- describe bag_joined:
- * -- bag_joined: {joined: {(bag1::key1: chararray, bag1::value1: chararray, bag2::key2: chararray, bag2::value2: int)}}
- * </code>
- * </p>
- *
- * @author wvaughan
- *
- */
-public class BagLeftOuterJoin extends AliasableEvalFunc<DataBag>
-{
-
- private static final String BAG_NAMES_PROPERTY = "BagLeftOuterJoin_BAG_NAMES";
- private static final String BAG_NAME_TO_JOIN_PREFIX_PROPERTY = "BagLeftOuterJoin_BAG_NAME_TO_JOIN_PREFIX";
- private static final String BAG_NAME_TO_SIZE_PROPERTY = "BagLeftOuterJoin_BAG_NAME_TO_SIZE_PROPERTY";
-
- ArrayList<String> bagNames;
- Map<String, String> bagNameToJoinKeyPrefix;
- Map<String, Integer> bagNameToSize;
-
- public BagLeftOuterJoin() {
-
- }
-
- @SuppressWarnings("unchecked")
- private void retrieveContextValues()
- {
- Properties properties = getInstanceProperties();
- bagNames = (ArrayList<String>) properties.get(BAG_NAMES_PROPERTY);
- bagNameToJoinKeyPrefix = (Map<String, String>) properties.get(BAG_NAME_TO_JOIN_PREFIX_PROPERTY);
- bagNameToSize = (Map<String, Integer>) properties.get(BAG_NAME_TO_SIZE_PROPERTY);
- }
-
- class JoinCollector
- {
- HashMap<Object, List<Tuple>> joinData;
-
- public void printJoinData() throws ExecException {
- printData(joinData);
- }
-
- public void printData(HashMap<Object, List<Tuple>> data) throws ExecException {
- for (Object o : data.keySet()) {
- System.out.println(o);
- for (Tuple t : data.get(o)) {
- System.out.println("\t"+t.toDelimitedString(", "));
- }
- }
- }
-
- public HashMap<Object, List<Tuple>> groupTuples(Iterable<Tuple> tuples, String keyName) throws ExecException {
- HashMap<Object, List<Tuple>> group = new HashMap<Object, List<Tuple>>();
- for (Tuple tuple : tuples) {
- Object key = getObject(tuple, keyName);
- if (!group.containsKey(key)) {
- group.put(key, new LinkedList<Tuple>());
- }
- group.get(key).add(tuple);
- }
- return group;
- }
-
- public HashMap<Object, List<Tuple>> insertNullTuples(HashMap<Object, List<Tuple>> groupedData, int tupleSize) throws ExecException {
- Tuple nullTuple = TupleFactory.getInstance().newTuple(tupleSize);
- for (int i=0; i<tupleSize; i++) {
- nullTuple.set(i, null);
- }
- for (Object key : joinData.keySet()) {
- if (!groupedData.containsKey(key)) {
- groupedData.put(key, Collections.singletonList(nullTuple));
- }
- }
- return groupedData;
- }
-
- public void joinTuples(Object key, List<Tuple> tuples) throws ExecException {
- List<Tuple> currentTuples = joinData.get(key);
- if (currentTuples != null) {
- List<Tuple> newTuples = new LinkedList<Tuple>();
- if (tuples != null) {
- for (Tuple t1 : currentTuples) {
- for (Tuple t2 : tuples) {
- Tuple t = TupleFactory.getInstance().newTuple();
- for (Object o : t1.getAll()) {
- t.append(o);
- }
- for (Object o : t2.getAll()) {
- t.append(o);
- }
- newTuples.add(t);
- }
- }
- }
- joinData.put(key, newTuples);
- }
- }
-
- public HashMap<Object, List<Tuple>> getJoinData() {
- return this.joinData;
- }
-
- public void setJoinData(HashMap<Object, List<Tuple>> joinData) {
- this.joinData = joinData;
- }
- }
-
- @Override
- public DataBag exec(Tuple input) throws IOException
- {
- retrieveContextValues();
-
- ArrayList<String> joinKeyNames = new ArrayList<String>();
- for (int i = 1; i < input.size(); i += 2) {
- joinKeyNames.add((String) input.get(i));
- }
-
- JoinCollector collector = new JoinCollector();
- // the first bag is the outer bag
- String leftBagName = bagNames.get(0);
- DataBag leftBag = getBag(input, leftBagName);
- String leftBagJoinKeyName = getPrefixedAliasName(bagNameToJoinKeyPrefix.get(leftBagName), joinKeyNames.get(0));
- collector.setJoinData(collector.groupTuples(leftBag, leftBagJoinKeyName));
- // now, for each additional bag, group up the tuples by the join key, then join them in
- if (bagNames.size() > 1) {
- for (int i = 1; i < bagNames.size(); i++) {
- String bagName = bagNames.get(i);
- DataBag bag = getBag(input, bagName);
- String joinKeyName = getPrefixedAliasName(bagNameToJoinKeyPrefix.get(bagName), joinKeyNames.get(i));
- int tupleSize = bagNameToSize.get(bagName);
- if (bag == null) throw new IOException("Error in instance: "+getInstanceName()
- + " with properties: " + getInstanceProperties()
- + " and tuple: " + input.toDelimitedString(", ")
- + " -- Expected bag, got null");
- HashMap<Object, List<Tuple>> groupedData = collector.groupTuples(bag, joinKeyName);
- // outer join, so go back in and add nulls;
- groupedData = collector.insertNullTuples(groupedData, tupleSize);
- for (Map.Entry<Object, List<Tuple>> entry : groupedData.entrySet()) {
- collector.joinTuples(entry.getKey(), entry.getValue());
- }
- }
- }
-
- // assemble output bag
- DataBag outputBag = BagFactory.getInstance().newDefaultBag();
- for (List<Tuple> tuples : collector.getJoinData().values()) {
- for (Tuple tuple : tuples) {
- outputBag.add(tuple);
- }
- }
-
- return outputBag;
- }
-
- @Override
- public Schema getOutputSchema(Schema input)
- {
- ArrayList<String> bagNames = new ArrayList<String>(input.size() / 2);
- Map<String, String> bagNameToJoinPrefix = new HashMap<String, String>(input.size() / 2);
- Map<String, Integer> bagNameToSize = new HashMap<String, Integer>(input.size() / 2);
- Schema outputSchema = null;
- Schema bagSchema = new Schema();
- try {
- int i = 0;
- // all even fields should be bags, odd fields are key names
- String bagName = null;
- String tupleName = null;
- for (FieldSchema outerField : input.getFields()) {
- if (i++ % 2 == 1)
- continue;
- bagName = outerField.alias;
- bagNames.add(bagName);
- if (bagName == null)
- bagName = "null";
- if (outerField.schema == null)
- throw new RuntimeException("Expected input format of (bag, 'field') pairs. "
- +"Did not receive a bag at index: "+i+", alias: "+bagName+". "
- +"Instead received type: "+DataType.findTypeName(outerField.type)+" in schema:"+input.toString());
- FieldSchema tupleField = outerField.schema.getField(0);
- tupleName = tupleField.alias;
- bagNameToJoinPrefix.put(bagName, getPrefixedAliasName(outerField.alias, tupleName));
- if (tupleField.schema == null) {
- log.error(String.format("could not get schema for inner tuple %s in bag %s", tupleName, bagName));
- } else {
- bagNameToSize.put(bagName, tupleField.schema.size());
- for (FieldSchema innerField : tupleField.schema.getFields()) {
- String innerFieldName = innerField.alias;
- if (innerFieldName == null)
- innerFieldName = "null";
- String outputFieldName = bagName + "::" + innerFieldName;
- bagSchema.add(new FieldSchema(outputFieldName, innerField.type));
- }
- }
- }
- outputSchema = new Schema(new Schema.FieldSchema("joined", bagSchema, DataType.BAG));
- log.debug("output schema: "+outputSchema.toString());
- } catch (FrontendException e) {
- e.printStackTrace();
- throw new RuntimeException(e);
- }
- Properties properties = getInstanceProperties();
- properties.put(BAG_NAMES_PROPERTY, bagNames);
- properties.put(BAG_NAME_TO_JOIN_PREFIX_PROPERTY, bagNameToJoinPrefix);
- properties.put(BAG_NAME_TO_SIZE_PROPERTY, bagNameToSize);
- return outputSchema;
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/bags/BagSplit.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/bags/BagSplit.java b/src/java/datafu/pig/bags/BagSplit.java
deleted file mode 100644
index 74d39d0..0000000
--- a/src/java/datafu/pig/bags/BagSplit.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.bags;
-
-import java.io.IOException;
-
-import org.apache.pig.EvalFunc;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * Splits a bag of tuples into a bag of bags, where the inner bags collectively contain
- * the tuples from the original bag. This can be used to split a bag into a set of smaller bags.
- * <p>
- * Example:
- * <pre>
- * {@code
- * define BagSplit datafu.pig.bags.BagSplit();
- *
- * -- input:
- * -- ({(1),(2),(3),(4),(5),(6),(7)})
- * -- ({(1),(2),(3),(4),(5)})
- * -- ({(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11)})
- * input = LOAD 'input' AS (B:bag{T:tuple(val1:INT,val2:INT)});
- *
- * -- ouput:
- * -- ({{(1),(2),(3),(4),(5)},{(6),(7)}})
- * -- ({{(1),(2),(3),(4),(5)},{(6),(7),(8),(9),(10)},{(11)}})
- * output = FOREACH input GENERATE BagSplit(5,B);
- * }
- * </pre>
- */
-public class BagSplit extends EvalFunc<DataBag>
-{
- private static final BagFactory bagFactory = BagFactory.getInstance();
- private static final TupleFactory tupleFactory = TupleFactory.getInstance();
-
- private final boolean appendBagNum;
-
- public BagSplit()
- {
- this.appendBagNum = false;
- }
-
- public BagSplit(String appendBagNum)
- {
- this.appendBagNum = Boolean.parseBoolean(appendBagNum);
- }
-
- @Override
- public DataBag exec(Tuple arg0) throws IOException
- {
- DataBag outputBag = bagFactory.newDefaultBag();
-
- Integer maxSize = (Integer)arg0.get(0);
-
- Object o = arg0.get(1);
- if (!(o instanceof DataBag))
- throw new RuntimeException("parameter must be a databag");
-
- DataBag inputBag = (DataBag)o;
-
- DataBag currentBag = null;
-
- int count = 0;
- int numBags = 0;
- for (Tuple tuple : inputBag)
- {
- if (currentBag == null)
- {
- currentBag = bagFactory.newDefaultBag();
- }
-
- currentBag.add(tuple);
- count++;
-
- if (count >= maxSize)
- {
- Tuple newTuple = tupleFactory.newTuple();
- newTuple.append(currentBag);
-
- if (this.appendBagNum)
- {
- newTuple.append(numBags);
- }
-
- numBags++;
-
- outputBag.add(newTuple);
-
- count = 0;
- currentBag = null;
- }
- }
-
- if (currentBag != null)
- {
- Tuple newTuple = tupleFactory.newTuple();
- newTuple.append(currentBag);
- if (this.appendBagNum)
- {
- newTuple.append(numBags);
- }
- outputBag.add(newTuple);
- }
-
- return outputBag;
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try {
- if (input.getField(0).type != DataType.INTEGER)
- {
- throw new RuntimeException("Expected first argument to be an INTEGER");
- }
-
- if (input.getField(1).type != DataType.BAG)
- {
- throw new RuntimeException("Expected second argument to be a BAG");
- }
-
- Schema tupleSchema = new Schema();
- tupleSchema.add(new Schema.FieldSchema("data", input.getField(1).schema.clone(), DataType.BAG));
-
- if (this.appendBagNum)
- {
- tupleSchema.add(new Schema.FieldSchema("index", DataType.INTEGER));
- }
-
- return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
- tupleSchema, DataType.BAG));
- }
- catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-}
[07/19] DATAFU-27 Migrate build system to Gradle
Posted by wv...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/sets/SetDifference.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/sets/SetDifference.java b/src/java/datafu/pig/sets/SetDifference.java
deleted file mode 100644
index b6469e9..0000000
--- a/src/java/datafu/pig/sets/SetDifference.java
+++ /dev/null
@@ -1,262 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.sets;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.PriorityQueue;
-
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.Tuple;
-
-/**
- * Computes the set difference of two or more bags. Duplicates are eliminated. <b>The input bags must be sorted.</b>
- *
- * <p>
- * If bags A and B are provided, then this computes A-B, i.e. all elements in A that are not in B.
- * If bags A, B and C are provided, then this computes A-B-C, i.e. all elements in A that are not in B or C.
- * </p>
- *
- * <p>
- * Example:
- * <pre>
- * {@code
- * define SetDifference datafu.pig.sets.SetDifference();
- *
- * -- input:
- * -- ({(1),(2),(3),(4),(5),(6)},{(3),(4)})
- * input = LOAD 'input' AS (B1:bag{T:tuple(val:int)},B2:bag{T:tuple(val:int)});
- *
- * input = FOREACH input {
- * B1 = ORDER B1 BY val ASC;
- * B2 = ORDER B2 BY val ASC;
- *
- * -- output:
- * -- ({(1),(2),(5),(6)})
- * GENERATE SetDifference(B1,B2);
- * }
- * }</pre>
- */
-public class SetDifference extends SetOperationsBase
-{
- private static final BagFactory bagFactory = BagFactory.getInstance();
-
- /**
- * Loads the data bags from the input tuple and puts them in a priority queue,
- * where ordering is determined by the data from the iterator for each bag.
- *
- * <p>
- * The bags are wrapped in a {@link Pair} object that is comparable on the data
- * currently available from the iterator.
- * These objects are ordered first by the data, then by the index within the tuple
- * the bag came from.
- * </p>
- *
- * @param input
- * @return
- * @throws IOException
- */
- private PriorityQueue<Pair> loadBags(Tuple input) throws IOException
- {
- PriorityQueue<Pair> pq = new PriorityQueue<Pair>(input.size());
-
- for (int i=0; i < input.size(); i++)
- {
- if (input.get(i) != null)
- {
- Iterator<Tuple> inputIterator = ((DataBag)input.get(i)).iterator();
- if(inputIterator.hasNext())
- {
- pq.add(new Pair(inputIterator,i));
- }
- }
- }
- return pq;
- }
-
- /**
- * Counts how many elements in the priority queue match the
- * element at the front of the queue, which should be from the first bag.
- *
- * @param pq priority queue
- * @return number of matches
- */
- public int countMatches(PriorityQueue<Pair> pq)
- {
- Pair nextPair = pq.peek();
- Tuple data = nextPair.data;
-
- // sanity check
- if (!nextPair.index.equals(0))
- {
- throw new RuntimeException("Expected next bag to have index 0");
- }
-
- int matches = 0;
- for (Pair p : pq) {
- if (data.equals(p.data))
- matches++;
- }
- // subtract 1 since element matches itself
- return matches - 1;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public DataBag exec(Tuple input) throws IOException
- {
- if (input.size() < 2)
- {
- throw new RuntimeException("Expected at least two inputs, but found " + input.size());
- }
-
- for (Object o : input)
- {
- if (o != null && !(o instanceof DataBag))
- {
- throw new RuntimeException("Inputs must be bags");
- }
- }
-
- DataBag outputBag = bagFactory.newDefaultBag();
-
- DataBag bag1 = (DataBag)input.get(0);
- DataBag bag2 = (DataBag)input.get(1);
-
- if (bag1 == null || bag1.size() == 0)
- {
- return outputBag;
- }
- // optimization
- else if (input.size() == 2 && (bag2 == null || bag2.size() == 0))
- {
- return bag1;
- }
-
- PriorityQueue<Pair> pq = loadBags(input);
-
- Tuple lastData = null;
-
- while (true)
- {
- Pair nextPair = pq.peek();
-
- // ignore data we've already encountered
- if (nextPair.data.compareTo(lastData) != 0)
- {
- // Only take data from the first bag, where there are no other
- // bags that have the same data.
- if (nextPair.index.equals(0) && countMatches(pq) == 0)
- {
- outputBag.add(nextPair.data);
- lastData = nextPair.data;
- }
- }
-
- Pair p = pq.poll();
-
- // only put the bag back into the queue if it still has data
- if (p.hasNext())
- {
- p.next();
- pq.offer(p);
- }
- else if (p.index.equals(0))
- {
- // stop when we exhaust all elements from the first bag
- break;
- }
- }
-
- return outputBag;
- }
-
- /**
- * A wrapper for the tuple iterator that implements comparable so it can be used in the priority queue.
- *
- * <p>
- * This is compared first on the data, then on the index the bag came from
- * in the input tuple.
- * </p>
- * @author mhayes
- *
- */
- private static class Pair implements Comparable<Pair>
- {
- private final Iterator<Tuple> it;
- private final Integer index;
- private Tuple data;
-
- /**
- * Constructs the {@link Pair}.
- *
- * @param it tuple iterator
- * @param index index within the tuple that the bag came from
- */
- public Pair(Iterator<Tuple> it, int index)
- {
- this.index = index;
- this.it = it;
- this.data = it.next();
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public int compareTo(Pair o)
- {
- int r = this.data.compareTo(o.data);
- if (r == 0)
- {
- return index.compareTo(o.index);
- }
- else
- {
- return r;
- }
- }
-
- public boolean hasNext()
- {
- return it.hasNext();
- }
-
- @SuppressWarnings("unchecked")
- public Tuple next()
- {
- Tuple nextData = it.next();
- // algorithm assumes data is in order
- if (data.compareTo(nextData) > 0)
- {
- throw new RuntimeException("Out of order!");
- }
- this.data = nextData;
- return this.data;
- }
-
- @Override
- public String toString()
- {
- return String.format("[%s within %d]",data,index);
- }
- }
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/sets/SetIntersect.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/sets/SetIntersect.java b/src/java/datafu/pig/sets/SetIntersect.java
deleted file mode 100644
index 0ba586e..0000000
--- a/src/java/datafu/pig/sets/SetIntersect.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.sets;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.PriorityQueue;
-
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-
-/**
- * Computes the set intersection of two or more bags. Duplicates are eliminated. <b>The input bags must be sorted.</b>
- * <p>
- * Example:
- * <pre>
- * {@code
- * define SetIntersect datafu.pig.sets.SetIntersect();
- *
- * -- input:
- * -- ({(1,10),(2,20),(3,30),(4,40)},{(2,20),(4,40),(8,80)})
- * input = LOAD 'input' AS (B1:bag{T:tuple(val1:int,val2:int)},B2:bag{T:tuple(val1:int,val2:int)});
- *
- * input = FOREACH input {
- * B1 = ORDER B1 BY val1 ASC, val2 ASC;
- * B2 = ORDER B2 BY val1 ASC, val2 ASC;
- *
- * -- output:
- * -- ({(2,20),(4,40)})
- * GENERATE SetIntersect(B1,B2);
- * }
- * }</pre>
- */
-public class SetIntersect extends SetOperationsBase
-{
- private static final BagFactory bagFactory = BagFactory.getInstance();
-
- static class pair implements Comparable<pair>
- {
- final Iterator<Tuple> it;
- Tuple data;
-
- pair(Iterator<Tuple> it)
- {
- this.it = it;
- this.data = it.next();
- }
-
- @Override
- public int compareTo(pair o)
- {
- return this.data.compareTo(o.data);
- }
- }
-
- private PriorityQueue<pair> load_bags(Tuple input) throws IOException
- {
- PriorityQueue<pair> pq = new PriorityQueue<pair>(input.size());
-
- for (int i=0; i < input.size(); i++) {
- Object o = input.get(i);
- if (!(o instanceof DataBag))
- throw new RuntimeException("parameters must be databags");
- Iterator<Tuple> inputIterator= ((DataBag) o).iterator();
- if(inputIterator.hasNext())
- pq.add(new pair(inputIterator));
- }
- return pq;
- }
-
- public boolean all_equal(PriorityQueue<pair> pq)
- {
- Object o = pq.peek().data;
- for (pair p : pq) {
- if (!o.equals(p.data))
- return false;
- }
- return true;
- }
-
- @Override
- public DataBag exec(Tuple input) throws IOException
- {
- DataBag outputBag = bagFactory.newDefaultBag();
- PriorityQueue<pair> pq = load_bags(input);
- if(pq.size() != input.size())
- return outputBag; // one or more input bags were empty
- Tuple last_data = null;
-
- while (true) {
- if (pq.peek().data.compareTo(last_data) != 0 && all_equal(pq)) {
- last_data = pq.peek().data;
- outputBag.add(last_data);
- }
-
- pair p = pq.poll();
- if (!p.it.hasNext())
- break;
- Tuple nextData = p.it.next();
- // algorithm assumes data is in order
- if (p.data.compareTo(nextData) > 0)
- {
- throw new RuntimeException("Out of order!");
- }
- p.data = nextData;
- pq.offer(p);
- }
-
- return outputBag;
- }
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/sets/SetOperationsBase.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/sets/SetOperationsBase.java b/src/java/datafu/pig/sets/SetOperationsBase.java
deleted file mode 100644
index c9997f8..0000000
--- a/src/java/datafu/pig/sets/SetOperationsBase.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.sets;
-
-import org.apache.pig.EvalFunc;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * Base class for set operations.
- *
- * @author "Matthew Hayes <mh...@linkedin.com>"
- *
- */
-public abstract class SetOperationsBase extends EvalFunc<DataBag>
-{
- @Override
- public Schema outputSchema(Schema input)
- {
- try {
- for (Schema.FieldSchema fieldSchema : input.getFields())
- {
- if (fieldSchema.type != DataType.BAG)
- {
- throw new RuntimeException("Expected a bag but got: " + DataType.findTypeName(fieldSchema.type));
- }
- }
-
- Schema bagSchema = input.getField(0).schema;
-
- Schema outputSchema = new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
- .getName()
- .toLowerCase(), input),
- bagSchema,
- DataType.BAG));
-
- return outputSchema;
- }
- catch (Exception e) {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/sets/SetUnion.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/sets/SetUnion.java b/src/java/datafu/pig/sets/SetUnion.java
deleted file mode 100644
index 49fd99c..0000000
--- a/src/java/datafu/pig/sets/SetUnion.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.sets;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-
-/**
- * Computes the set union of two or more bags. Duplicates are eliminated.
- * <p>
- * Example:
- * <pre>
- * {@code
- * define SetUnion datafu.pig.sets.SetUnion();
- *
- * -- input:
- * -- ({(2,20),(3,30),(4,40)},{(1,10),(2,20),(4,40),(8,80)})
- * input = LOAD 'input' AS (B1:bag{T:tuple(val1:int,val2:int)},B2:bag{T:tuple(val1:int,val2:int)});
- *
- * -- output:
- * -- ({(2,20),(3,30),(4,40),(1,10),(8,80)})
- * output = FOREACH input GENERATE SetUnion(B1,B2);
- * }
- * </pre>
- */
-public class SetUnion extends SetOperationsBase
-{
- private static final BagFactory bagFactory = BagFactory.getInstance();
- private static final TupleFactory tupleFactory = TupleFactory.getInstance();
-
- @Override
- public DataBag exec(Tuple input) throws IOException
- {
- DataBag outputBag = bagFactory.newDistinctBag();
-
- try {
- for (int i=0; i < input.size(); i++) {
- Object o = input.get(i);
- if (!(o instanceof DataBag))
- throw new RuntimeException("parameters must be databags");
-
- DataBag inputBag = (DataBag) o;
- for (Tuple elem : inputBag) {
- outputBag.add(elem);
- }
- }
-
- return outputBag;
- }
- catch (Exception e) {
- throw new IOException(e);
- }
- }
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/sets/package-info.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/sets/package-info.java b/src/java/datafu/pig/sets/package-info.java
deleted file mode 100644
index 2c5b087..0000000
--- a/src/java/datafu/pig/sets/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * UDFs for set operations such as intersect and union.
- */
-package datafu.pig.sets;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/DoubleVAR.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/DoubleVAR.java b/src/java/datafu/pig/stats/DoubleVAR.java
deleted file mode 100644
index ab8eed2..0000000
--- a/src/java/datafu/pig/stats/DoubleVAR.java
+++ /dev/null
@@ -1,373 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.pig.Accumulator;
-import org.apache.pig.Algebraic;
-import org.apache.pig.EvalFunc;
-import org.apache.pig.FuncSpec;
-import org.apache.pig.PigException;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataByteArray;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.backend.executionengine.ExecException;
-
-
-/**
-* Use {@link VAR}
-*/
-public class DoubleVAR extends EvalFunc<Double> implements Algebraic, Accumulator<Double> {
- private static TupleFactory mTupleFactory = TupleFactory.getInstance();
-
- @Override
- public Double exec(Tuple input) throws IOException {
- try {
- Double sum = sum(input);
- Double sumSquare = sumSquare(input);
-
- if(sum == null) {
- // either we were handed an empty bag or a bag
- // filled with nulls - return null in this case
- return null;
- }
- long count = count(input);
-
- Double var = null;
- if (count > 0){
- Double avg = new Double(sum / count);
- Double avgSquare = new Double(sumSquare / count);
- var = avgSquare - avg*avg;
- }
-
- return var;
- } catch (ExecException ee) {
- throw ee;
- }
- }
-
- public String getInitial() {
- return Initial.class.getName();
- }
-
- public String getIntermed() {
- return Intermediate.class.getName();
- }
-
- public String getFinal() {
- return Final.class.getName();
- }
-
- static public class Initial extends EvalFunc<Tuple> {
- @Override
- public Tuple exec(Tuple input) throws IOException {
- Tuple t = mTupleFactory.newTuple(3);
- try {
- // input is a bag with one tuple containing
- // the column we are trying to get variance
- DataBag bg = (DataBag) input.get(0);
- Double d = null;
- Iterator<Tuple> iter = bg.iterator();
- if(iter.hasNext()) {
- Tuple tp = iter.next();
- d = (Double)tp.get(0);
- }
-
- if (iter.hasNext())
- {
- throw new RuntimeException("Expected only one tuple in bag");
- }
-
- if (d == null){
- t.set(0, null);
- t.set(1, null);
- t.set(2, 0L);
- }
- else {
- t.set(0, d);
- t.set(1, d*d);
- t.set(2, 1L);
- }
- return t;
- } catch(NumberFormatException nfe) {
- nfe.printStackTrace();
- // invalid input,
- // treat this input as null
- try {
- t.set(0, null);
- t.set(1, null);
- t.set(2, 0L);
- } catch (ExecException e) {
- throw e;
- }
- return t;
- } catch (ExecException ee) {
- ee.printStackTrace();
- throw ee;
- } catch (Exception e) {
- e.printStackTrace();
- int errCode = 2106;
- String msg = "Error while computing variance in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
- }
-
- }
- }
-
- static public class Intermediate extends EvalFunc<Tuple> {
- @Override
- public Tuple exec(Tuple input) throws IOException {
- try {
- DataBag b = (DataBag)input.get(0);
- return combine(b);
- } catch (ExecException ee) {
- throw ee;
- } catch (Exception e) {
- int errCode = 2106;
- String msg = "Error while computing variacne in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
-
- }
- }
- }
-
- static public class Final extends EvalFunc<Double> {
- @Override
- public Double exec(Tuple input) throws IOException {
- try {
- DataBag b = (DataBag)input.get(0);
- Tuple combined = combine(b);
-
- Double sum = (Double)combined.get(0);
- Double sumSquare = (Double)combined.get(1);
- if(sum == null) {
- return null;
- }
- Long count = (Long)combined.get(2);
-
- Double var = null;
-
- if (count > 0) {
- Double avg = new Double(sum / count);
- Double avgSquare = new Double(sumSquare / count);
- var = avgSquare - avg*avg;
- }
- return var;
- } catch (ExecException ee) {
- throw ee;
- } catch (Exception e) {
- int errCode = 2106;
- String msg = "Error while computing variance in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
- }
- }
- }
-
- static protected Tuple combine(DataBag values) throws ExecException{
- double sum = 0;
- double sumSquare = 0;
- long totalCount = 0;
-
- // combine is called from Intermediate and Final
- // In either case, Initial would have been called
- // before and would have sent in valid tuples
- // Hence we don't need to check if incoming bag
- // is empty
-
- Tuple output = mTupleFactory.newTuple(3);
- boolean sawNonNull = false;
- for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
- Tuple t = it.next();
- Double d = (Double)t.get(0);
- Double dSquare = (Double)t.get(1);
- Long count = (Long)t.get(2);
-
- // we count nulls in var as contributing 0
- // a departure from SQL for performance of
- // COUNT() which implemented by just inspecting
- // size of the bag
- if(d == null) {
- d = 0.0;
- dSquare = 0.0;
- } else {
- sawNonNull = true;
- }
- sum += d;
- sumSquare += dSquare;
- totalCount += count;
- }
- if(sawNonNull) {
- output.set(0, new Double(sum));
- output.set(1, new Double(sumSquare));
- } else {
- output.set(0, null);
- output.set(1, null);
- }
- output.set(2, Long.valueOf(totalCount));
- return output;
- }
-
- static protected long count(Tuple input) throws ExecException {
- DataBag values = (DataBag)input.get(0);
- long cnt = 0;
- Iterator<Tuple> it = values.iterator();
- while (it.hasNext()){
- Tuple t = (Tuple)it.next();
- if (t != null && t.size() > 0 && t.get(0) != null)
- cnt ++;
- }
-
- return cnt;
- }
-
- static protected Double sum(Tuple input) throws ExecException, IOException {
- DataBag values = (DataBag)input.get(0);
-
- // if we were handed an empty bag, return NULL
- if(values.size() == 0) {
- return null;
- }
-
- double sum = 0;
- boolean sawNonNull = false;
- for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
- Tuple t = it.next();
- try{
- Double d= (Double)t.get(0);
- if (d == null) continue;
- sawNonNull = true;
- sum += d;
- }catch(RuntimeException exp) {
- int errCode = 2103;
- String msg = "Problem while computing sum of values.";
- throw new ExecException(msg, errCode, PigException.BUG, exp);
- }
- }
-
- if(sawNonNull) {
- return new Double(sum);
- } else {
- return null;
- }
- }
-
- static protected Double sumSquare(Tuple input) throws ExecException, IOException {
- DataBag values = (DataBag)input.get(0);
-
- // if we were handed an empty bag, return NULL
- if(values.size() == 0) {
- return null;
- }
-
- double sumSquare = 0;
- boolean sawNonNull = false;
- for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
- Tuple t = it.next();
- try{
- Double d = (Double)t.get(0);
- if (d == null) continue;
- sawNonNull = true;
- sumSquare += d*d;
- }catch(RuntimeException exp) {
- int errCode = 2103;
- String msg = "Problem while computing sum of squared values.";
- throw new ExecException(msg, errCode, PigException.BUG, exp);
- }
- }
-
- if(sawNonNull) {
- return new Double(sumSquare);
- } else {
- return null;
- }
- }
-
- @Override
- public Schema outputSchema(Schema input) {
- return new Schema(new Schema.FieldSchema(null, DataType.DOUBLE));
- }
-
- /* Accumulator interface implementation */
- private Double intermediateSumSquare = null;
- private Double intermediateSum = null;
- private Long intermediateCount = null;
-
- @Override
- public void accumulate(Tuple b) throws IOException {
- try {
- Double sum = sum(b);
- if(sum == null) {
- return;
- }
-
- Double sumSquare = sumSquare(b);
- if(sumSquare == null) {
- return;
- }
-
- // set default values
- if (intermediateSum == null || intermediateCount == null) {
- intermediateSumSquare = 0.0;
- intermediateSum = 0.0;
- intermediateCount = (long) 0;
- }
-
- long count = (Long)count(b);
-
- if (count > 0) {
- intermediateCount += count;
- intermediateSum += sum;
- intermediateSumSquare += sumSquare;
- }
- } catch (ExecException ee) {
- throw ee;
- } catch (Exception e) {
- int errCode = 2106;
- String msg = "Error while computing variance in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
- }
- }
-
- @Override
- public void cleanup() {
- intermediateSumSquare = null;
- intermediateSum = null;
- intermediateCount = null;
- }
-
- @Override
- public Double getValue() {
- Double var = null;
- if (intermediateCount != null && intermediateCount > 0) {
- Double avg = new Double(intermediateSum / intermediateCount);
- Double avgSquare = new Double(intermediateSumSquare / intermediateCount);
- var = avgSquare - avg*avg;
- }
- return var;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/FloatVAR.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/FloatVAR.java b/src/java/datafu/pig/stats/FloatVAR.java
deleted file mode 100644
index 8a697e0..0000000
--- a/src/java/datafu/pig/stats/FloatVAR.java
+++ /dev/null
@@ -1,375 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.pig.Accumulator;
-import org.apache.pig.Algebraic;
-import org.apache.pig.EvalFunc;
-import org.apache.pig.FuncSpec;
-import org.apache.pig.PigException;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataByteArray;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.backend.executionengine.ExecException;
-
-
-/**
-* Use {@link VAR}
-*/
-public class FloatVAR extends EvalFunc<Double> implements Algebraic, Accumulator<Double> {
- private static TupleFactory mTupleFactory = TupleFactory.getInstance();
-
- @Override
- public Double exec(Tuple input) throws IOException {
- try {
- Double sum = sum(input);
- Double sumSquare = sumSquare(input);
-
- if(sum == null) {
- // either we were handed an empty bag or a bag
- // filled with nulls - return null in this case
- return null;
- }
- long count = count(input);
-
- Double var = null;
- if (count > 0){
- Double avg = new Double(sum / count);
- Double avgSquare = new Double(sumSquare / count);
- var = avgSquare - avg*avg;
- }
-
- return var;
- } catch (ExecException ee) {
- throw ee;
- }
- }
-
- public String getInitial() {
- return Initial.class.getName();
- }
-
- public String getIntermed() {
- return Intermediate.class.getName();
- }
-
- public String getFinal() {
- return Final.class.getName();
- }
-
- static public class Initial extends EvalFunc<Tuple> {
- @Override
- public Tuple exec(Tuple input) throws IOException {
- Tuple t = mTupleFactory.newTuple(3);
- try {
- // input is a bag with one tuple containing
- // the column we are trying to get variance
- DataBag bg = (DataBag) input.get(0);
- Float f = null;
- Iterator<Tuple> iter = bg.iterator();
- if(iter.hasNext()) {
- Tuple tp = iter.next();
- f = (Float)tp.get(0);
- }
-
- if (iter.hasNext())
- {
- throw new RuntimeException("Expected only one tuple in bag");
- }
-
- Double d = f!= null ? new Double(f): null;
-
- if (f == null){
- t.set(0, null);
- t.set(1, null);
- t.set(2, 0L);
- }
- else {
- t.set(0, d);
- t.set(1, d*d);
- t.set(2, 1L);
- }
- return t;
- } catch(NumberFormatException nfe) {
- nfe.printStackTrace();
- // invalid input,
- // treat this input as null
- try {
- t.set(0, null);
- t.set(1, null);
- t.set(2, 0L);
- } catch (ExecException e) {
- throw e;
- }
- return t;
- } catch (ExecException ee) {
- ee.printStackTrace();
- throw ee;
- } catch (Exception e) {
- e.printStackTrace();
- int errCode = 2106;
- String msg = "Error while computing variance in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
- }
-
- }
- }
-
- static public class Intermediate extends EvalFunc<Tuple> {
- @Override
- public Tuple exec(Tuple input) throws IOException {
- try {
- DataBag b = (DataBag)input.get(0);
- return combine(b);
- } catch (ExecException ee) {
- throw ee;
- } catch (Exception e) {
- int errCode = 2106;
- String msg = "Error while computing variacne in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
-
- }
- }
- }
-
- static public class Final extends EvalFunc<Double> {
- @Override
- public Double exec(Tuple input) throws IOException {
- try {
- DataBag b = (DataBag)input.get(0);
- Tuple combined = combine(b);
-
- Double sum = (Double)combined.get(0);
- Double sumSquare = (Double)combined.get(1);
- if(sum == null) {
- return null;
- }
- Long count = (Long)combined.get(2);
-
- Double var = null;
-
- if (count > 0) {
- Double avg = new Double(sum / count);
- Double avgSquare = new Double(sumSquare / count);
- var = avgSquare - avg*avg;
- }
- return var;
- } catch (ExecException ee) {
- throw ee;
- } catch (Exception e) {
- int errCode = 2106;
- String msg = "Error while computing variance in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
- }
- }
- }
-
- static protected Tuple combine(DataBag values) throws ExecException{
- double sum = 0;
- double sumSquare = 0;
- long totalCount = 0;
-
- // combine is called from Intermediate and Final
- // In either case, Initial would have been called
- // before and would have sent in valid tuples
- // Hence we don't need to check if incoming bag
- // is empty
-
- Tuple output = mTupleFactory.newTuple(3);
- boolean sawNonNull = false;
- for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
- Tuple t = it.next();
- Double d = (Double)t.get(0);
- Double dSquare = (Double)t.get(1);
- Long count = (Long)t.get(2);
-
- // we count nulls in var as contributing 0
- // a departure from SQL for performance of
- // COUNT() which implemented by just inspecting
- // size of the bag
- if(d == null) {
- d = 0.0;
- dSquare = 0.0;
- } else {
- sawNonNull = true;
- }
- sum += d;
- sumSquare += dSquare;
- totalCount += count;
- }
- if(sawNonNull) {
- output.set(0, new Double(sum));
- output.set(1, new Double(sumSquare));
- } else {
- output.set(0, null);
- output.set(1, null);
- }
- output.set(2, Long.valueOf(totalCount));
- return output;
- }
-
- static protected long count(Tuple input) throws ExecException {
- DataBag values = (DataBag)input.get(0);
- long cnt = 0;
- Iterator<Tuple> it = values.iterator();
- while (it.hasNext()){
- Tuple t = (Tuple)it.next();
- if (t != null && t.size() > 0 && t.get(0) != null)
- cnt ++;
- }
-
- return cnt;
- }
-
- static protected Double sum(Tuple input) throws ExecException, IOException {
- DataBag values = (DataBag)input.get(0);
-
- // if we were handed an empty bag, return NULL
- if(values.size() == 0) {
- return null;
- }
-
- double sum = 0;
- boolean sawNonNull = false;
- for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
- Tuple t = it.next();
- try{
- Float f= (Float)t.get(0);
- if (f == null) continue;
- sawNonNull = true;
- sum += f;
- }catch(RuntimeException exp) {
- int errCode = 2103;
- String msg = "Problem while computing sum of values.";
- throw new ExecException(msg, errCode, PigException.BUG, exp);
- }
- }
-
- if(sawNonNull) {
- return new Double(sum);
- } else {
- return null;
- }
- }
-
- static protected Double sumSquare(Tuple input) throws ExecException, IOException {
- DataBag values = (DataBag)input.get(0);
-
- // if we were handed an empty bag, return NULL
- if(values.size() == 0) {
- return null;
- }
-
- double sumSquare = 0;
- boolean sawNonNull = false;
- for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
- Tuple t = it.next();
- try{
- Float f = (Float)t.get(0);
- if (f == null) continue;
- sawNonNull = true;
- sumSquare += f*f;
- }catch(RuntimeException exp) {
- int errCode = 2103;
- String msg = "Problem while computing sum of squared values.";
- throw new ExecException(msg, errCode, PigException.BUG, exp);
- }
- }
-
- if(sawNonNull) {
- return new Double(sumSquare);
- } else {
- return null;
- }
- }
-
- @Override
- public Schema outputSchema(Schema input) {
- return new Schema(new Schema.FieldSchema(null, DataType.DOUBLE));
- }
-
- /* Accumulator interface implementation */
- private Double intermediateSumSquare = null;
- private Double intermediateSum = null;
- private Long intermediateCount = null;
-
- @Override
- public void accumulate(Tuple b) throws IOException {
- try {
- Double sum = sum(b);
- if(sum == null) {
- return;
- }
-
- Double sumSquare = sumSquare(b);
- if(sumSquare == null) {
- return;
- }
-
- // set default values
- if (intermediateSum == null || intermediateCount == null) {
- intermediateSumSquare = 0.0;
- intermediateSum = 0.0;
- intermediateCount = (long) 0;
- }
-
- long count = (Long)count(b);
-
- if (count > 0) {
- intermediateCount += count;
- intermediateSum += sum;
- intermediateSumSquare += sumSquare;
- }
- } catch (ExecException ee) {
- throw ee;
- } catch (Exception e) {
- int errCode = 2106;
- String msg = "Error while computing variance in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
- }
- }
-
- @Override
- public void cleanup() {
- intermediateSumSquare = null;
- intermediateSum = null;
- intermediateCount = null;
- }
-
- @Override
- public Double getValue() {
- Double var = null;
- if (intermediateCount != null && intermediateCount > 0) {
- Double avg = new Double(intermediateSum / intermediateCount);
- Double avgSquare = new Double(intermediateSumSquare / intermediateCount);
- var = avgSquare - avg*avg;
- }
- return var;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/HyperLogLogPlusPlus.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/HyperLogLogPlusPlus.java b/src/java/datafu/pig/stats/HyperLogLogPlusPlus.java
deleted file mode 100644
index 0ebd94f..0000000
--- a/src/java/datafu/pig/stats/HyperLogLogPlusPlus.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats;
-
-import java.io.IOException;
-
-import org.apache.pig.AccumulatorEvalFunc;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-
-/**
- * A UDF that applies the HyperLogLog++ cardinality estimation algorithm.
- *
- * <p>
- * This uses the implementation of HyperLogLog++ from <a href="https://github.com/addthis/stream-lib" target="_blank">stream-lib</a>.
- * The HyperLogLog++ algorithm is an enhanced version of HyperLogLog as described in
- * <a href="http://static.googleusercontent.com/external_content/untrusted_dlcp/research.google.com/en/us/pubs/archive/40671.pdf">here</a>.
- * </p>
- *
- * <p>
- * This is a streaming implementation, and therefore the input data does not need to be sorted.
- * </p>
- *
- * @author mhayes
- *
- */
-public class HyperLogLogPlusPlus extends AccumulatorEvalFunc<Long>
-{
- private com.clearspring.analytics.stream.cardinality.HyperLogLogPlus estimator;
-
- private final int p;
-
- /**
- * Constructs a HyperLogLog++ estimator.
- */
- public HyperLogLogPlusPlus()
- {
- this("20");
- }
-
- /**
- * Constructs a HyperLogLog++ estimator.
- *
- * @param p precision value
- */
- public HyperLogLogPlusPlus(String p)
- {
- this.p = Integer.parseInt(p);
- cleanup();
- }
-
- @Override
- public void accumulate(Tuple arg0) throws IOException
- {
- DataBag inputBag = (DataBag)arg0.get(0);
- for (Tuple t : inputBag)
- {
- estimator.offer(t);
- }
- }
-
- @Override
- public void cleanup()
- {
- this.estimator = new com.clearspring.analytics.stream.cardinality.HyperLogLogPlus(p);
- }
-
- @Override
- public Long getValue()
- {
- return this.estimator.cardinality();
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try {
- if (input.size() != 1)
- {
- throw new RuntimeException("Expected input to have only a single field");
- }
-
- Schema.FieldSchema inputFieldSchema = input.getField(0);
-
- if (inputFieldSchema.type != DataType.BAG)
- {
- throw new RuntimeException("Expected a BAG as input");
- }
-
- return new Schema(new Schema.FieldSchema(null, DataType.LONG));
- }
- catch (FrontendException e) {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/IntVAR.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/IntVAR.java b/src/java/datafu/pig/stats/IntVAR.java
deleted file mode 100644
index 0f34d8e..0000000
--- a/src/java/datafu/pig/stats/IntVAR.java
+++ /dev/null
@@ -1,372 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.pig.Accumulator;
-import org.apache.pig.Algebraic;
-import org.apache.pig.EvalFunc;
-import org.apache.pig.FuncSpec;
-import org.apache.pig.PigException;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataByteArray;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.backend.executionengine.ExecException;
-
-
-/**
-* Use {@link VAR}
-*/
-public class IntVAR extends EvalFunc<Double> implements Algebraic, Accumulator<Double> {
- private static TupleFactory mTupleFactory = TupleFactory.getInstance();
-
- @Override
- public Double exec(Tuple input) throws IOException {
- try {
- Long sum = sum(input);
- Long sumSquare = sumSquare(input);
-
- if(sum == null) {
- // either we were handed an empty bag or a bag
- // filled with nulls - return null in this case
- return null;
- }
- long count = count(input);
-
- Double var = null;
- if (count > 0){
- Double avg = new Double((double)sum / count);
- Double avgSquare = new Double((double)sumSquare / count);
- var = avgSquare - avg*avg;
- }
-
- return var;
- } catch (ExecException ee) {
- throw ee;
- }
- }
-
- public String getInitial() {
- return Initial.class.getName();
- }
-
- public String getIntermed() {
- return Intermediate.class.getName();
- }
-
- public String getFinal() {
- return Final.class.getName();
- }
-
- static public class Initial extends EvalFunc<Tuple> {
- @Override
- public Tuple exec(Tuple input) throws IOException {
- Tuple t = mTupleFactory.newTuple(3);
- try {
- // input is a bag with one tuple containing
- // the column we are trying to get variance
- DataBag bg = (DataBag) input.get(0);
- Integer i = null;
- Iterator<Tuple> iter = bg.iterator();
- if(iter.hasNext()) {
- Tuple tp = iter.next();
- i = (Integer)tp.get(0);
- }
-
- if (iter.hasNext())
- {
- throw new RuntimeException("Expected only one tuple in bag");
- }
-
- if (i == null) {
- t.set(2, 0L);
- t.set(0, null);
- t.set(1, null);
- }
- else {
- t.set(2, 1L);
- t.set(0, (long)i);
- t.set(1, (long)i*i);
- }
- return t;
- } catch(NumberFormatException nfe) {
- nfe.printStackTrace();
- // invalid input,
- // treat this input as null
- try {
- t.set(0, null);
- t.set(1, null);
- t.set(2, 0L);
- } catch (ExecException e) {
- throw e;
- }
- return t;
- } catch (ExecException ee) {
- ee.printStackTrace();
- throw ee;
- } catch (Exception e) {
- e.printStackTrace();
- int errCode = 2106;
- String msg = "Error while computing variance in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
- }
-
- }
- }
-
- static public class Intermediate extends EvalFunc<Tuple> {
- @Override
- public Tuple exec(Tuple input) throws IOException {
- try {
- DataBag b = (DataBag)input.get(0);
- return combine(b);
- } catch (ExecException ee) {
- throw ee;
- } catch (Exception e) {
- int errCode = 2106;
- String msg = "Error while computing variacne in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
- }
- }
- }
-
- static public class Final extends EvalFunc<Double> {
- @Override
- public Double exec(Tuple input) throws IOException {
- try {
- DataBag b = (DataBag)input.get(0);
- Tuple combined = combine(b);
-
- Long sum = (Long)combined.get(0);
- Long sumSquare = (Long)combined.get(1);
- if(sum == null) {
- return null;
- }
- Long count = (Long)combined.get(2);
-
- Double var = null;
-
- if (count > 0) {
- Double avg = new Double((double)sum / count);
- Double avgSquare = new Double((double)sumSquare / count);
- var = avgSquare - avg*avg;
- }
- return var;
- } catch (ExecException ee) {
- throw ee;
- } catch (Exception e) {
- int errCode = 2106;
- String msg = "Error while computing variance in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
- }
- }
- }
-
- static protected Tuple combine(DataBag values) throws ExecException{
- long sum = 0;
- long sumSquare = 0;
- long totalCount = 0;
-
- // combine is called from Intermediate and Final
- // In either case, Initial would have been called
- // before and would have sent in valid tuples
- // Hence we don't need to check if incoming bag
- // is empty
-
- Tuple output = mTupleFactory.newTuple(3);
- boolean sawNonNull = false;
- for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
- Tuple t = it.next();
- Long i = (Long)t.get(0);
- Long iSquare = (Long)t.get(1);
- Long count = (Long)t.get(2);
-
- // we count nulls in var as contributing 0
- // a departure from SQL for performance of
- // COUNT() which implemented by just inspecting
- // size of the bag
- if(i == null) {
- i = (long)0;
- iSquare = (long)0;
- } else {
- sawNonNull = true;
- }
- sum += i;
- sumSquare += iSquare;
- totalCount += count;
- }
- if(sawNonNull) {
- output.set(0, new Long(sum));
- output.set(1, new Long(sumSquare));
- } else {
- output.set(0, null);
- output.set(1, null);
- }
- output.set(2, Long.valueOf(totalCount));
- return output;
- }
-
- static protected long count(Tuple input) throws ExecException {
- DataBag values = (DataBag)input.get(0);
- long cnt = 0;
- Iterator<Tuple> it = values.iterator();
- while (it.hasNext()){
- Tuple t = (Tuple)it.next();
- if (t != null && t.size() > 0 && t.get(0) != null)
- cnt ++;
- }
-
- return cnt;
- }
-
- static protected Long sum(Tuple input) throws ExecException, IOException {
- DataBag values = (DataBag)input.get(0);
-
- // if we were handed an empty bag, return NULL
- if(values.size() == 0) {
- return null;
- }
-
- long sum = 0;
- boolean sawNonNull = false;
- for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
- Tuple t = it.next();
- try{
- Integer i = (Integer)t.get(0);
- if (i == null) continue;
- sawNonNull = true;
- sum += i;
- }catch(RuntimeException exp) {
- int errCode = 2103;
- String msg = "Problem while computing sum of values.";
- throw new ExecException(msg, errCode, PigException.BUG, exp);
- }
- }
-
- if(sawNonNull) {
- return new Long(sum);
- } else {
- return null;
- }
- }
-
- static protected Long sumSquare(Tuple input) throws ExecException, IOException {
- DataBag values = (DataBag)input.get(0);
-
- // if we were handed an empty bag, return NULL
- if(values.size() == 0) {
- return null;
- }
-
- long sumSquare = 0;
- boolean sawNonNull = false;
- for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
- Tuple t = it.next();
- try{
- Integer i = (Integer)t.get(0);
- if (i == null) continue;
- sawNonNull = true;
- sumSquare += i*i;
- }catch(RuntimeException exp) {
- int errCode = 2103;
- String msg = "Problem while computing sum of squared values.";
- throw new ExecException(msg, errCode, PigException.BUG, exp);
- }
- }
-
- if(sawNonNull) {
- return new Long(sumSquare);
- } else {
- return null;
- }
- }
-
- @Override
- public Schema outputSchema(Schema input) {
- return new Schema(new Schema.FieldSchema(null, DataType.DOUBLE));
- }
-
- /* Accumulator interface implementation */
- private Long intermediateSumSquare = null;
- private Long intermediateSum = null;
- private Long intermediateCount = null;
-
- @Override
- public void accumulate(Tuple b) throws IOException {
- try {
- Long sum = sum(b);
- if(sum == null) {
- return;
- }
-
- Long sumSquare = sumSquare(b);
- if(sumSquare == null) {
- return;
- }
-
- // set default values
- if (intermediateSum == null || intermediateCount == null) {
- intermediateSumSquare = (long) 0;
- intermediateSum = (long) 0;
- intermediateCount = (long) 0;
- }
-
- long count = (Long)count(b);
-
- if (count > 0) {
- intermediateCount += count;
- intermediateSum += sum;
- intermediateSumSquare += sumSquare;
- }
- } catch (ExecException ee) {
- throw ee;
- } catch (Exception e) {
- int errCode = 2106;
- String msg = "Error while computing variance in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
- }
- }
-
- @Override
- public void cleanup() {
- intermediateSumSquare = null;
- intermediateSum = null;
- intermediateCount = null;
- }
-
- @Override
- public Double getValue() {
- Double var = null;
- if (intermediateCount != null && intermediateCount > 0) {
- Double avg = new Double((double)intermediateSum / intermediateCount);
- Double avgSquare = new Double((double)intermediateSumSquare / intermediateCount);
- var = avgSquare - avg*avg;
- }
- return var;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/LongVAR.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/LongVAR.java b/src/java/datafu/pig/stats/LongVAR.java
deleted file mode 100644
index 561f15a..0000000
--- a/src/java/datafu/pig/stats/LongVAR.java
+++ /dev/null
@@ -1,373 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.pig.Accumulator;
-import org.apache.pig.Algebraic;
-import org.apache.pig.EvalFunc;
-import org.apache.pig.FuncSpec;
-import org.apache.pig.PigException;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataByteArray;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.backend.executionengine.ExecException;
-
-
-/**
-* Use {@link VAR}
-*/
-public class LongVAR extends EvalFunc<Double> implements Algebraic, Accumulator<Double> {
- private static TupleFactory mTupleFactory = TupleFactory.getInstance();
-
- @Override
- public Double exec(Tuple input) throws IOException {
- try {
- Long sum = sum(input);
- Long sumSquare = sumSquare(input);
-
- if(sum == null) {
- // either we were handed an empty bag or a bag
- // filled with nulls - return null in this case
- return null;
- }
- long count = count(input);
-
- Double var = null;
- if (count > 0){
- Double avg = new Double((double)sum / count);
- Double avgSquare = new Double((double)sumSquare / count);
- var = avgSquare - avg*avg;
- }
-
- return var;
- } catch (ExecException ee) {
- throw ee;
- }
- }
-
- public String getInitial() {
- return Initial.class.getName();
- }
-
- public String getIntermed() {
- return Intermediate.class.getName();
- }
-
- public String getFinal() {
- return Final.class.getName();
- }
-
- static public class Initial extends EvalFunc<Tuple> {
- @Override
- public Tuple exec(Tuple input) throws IOException {
- Tuple t = mTupleFactory.newTuple(3);
- try {
- // input is a bag with one tuple containing
- // the column we are trying to get variance
- DataBag bg = (DataBag) input.get(0);
- Long l = null;
- Iterator<Tuple> iter = bg.iterator();
- if(iter.hasNext()) {
- Tuple tp = iter.next();
- l = (Long)tp.get(0);
- }
-
- if (iter.hasNext())
- {
- throw new RuntimeException("Expected only one tuple in bag");
- }
-
- if (l == null) {
- t.set(0, null);
- t.set(1, null);
- t.set(2, 0L);
- }
- else {
- t.set(0, l);
- t.set(1, l*l);
- t.set(2, 1L);
- }
- return t;
- } catch(NumberFormatException nfe) {
- nfe.printStackTrace();
- // invalid input,
- // treat this input as null
- try {
- t.set(0, null);
- t.set(1, null);
- t.set(2, 0L);
- } catch (ExecException e) {
- throw e;
- }
- return t;
- } catch (ExecException ee) {
- ee.printStackTrace();
- throw ee;
- } catch (Exception e) {
- e.printStackTrace();
- int errCode = 2106;
- String msg = "Error while computing variance in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
- }
-
- }
- }
-
- static public class Intermediate extends EvalFunc<Tuple> {
- @Override
- public Tuple exec(Tuple input) throws IOException {
- try {
- DataBag b = (DataBag)input.get(0);
- return combine(b);
- } catch (ExecException ee) {
- throw ee;
- } catch (Exception e) {
- int errCode = 2106;
- String msg = "Error while computing variacne in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
-
- }
- }
- }
-
- static public class Final extends EvalFunc<Double> {
- @Override
- public Double exec(Tuple input) throws IOException {
- try {
- DataBag b = (DataBag)input.get(0);
- Tuple combined = combine(b);
-
- Long sum = (Long)combined.get(0);
- Long sumSquare = (Long)combined.get(1);
- if(sum == null) {
- return null;
- }
- Long count = (Long)combined.get(2);
-
- Double var = null;
-
- if (count > 0) {
- Double avg = new Double((double)sum / count);
- Double avgSquare = new Double((double)sumSquare / count);
- var = avgSquare - avg*avg;
- }
- return var;
- } catch (ExecException ee) {
- throw ee;
- } catch (Exception e) {
- int errCode = 2106;
- String msg = "Error while computing variance in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
- }
- }
- }
-
- static protected Tuple combine(DataBag values) throws ExecException{
- long sum = 0;
- long sumSquare = 0;
- long totalCount = 0;
-
- // combine is called from Intermediate and Final
- // In either case, Initial would have been called
- // before and would have sent in valid tuples
- // Hence we don't need to check if incoming bag
- // is empty
-
- Tuple output = mTupleFactory.newTuple(3);
- boolean sawNonNull = false;
- for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
- Tuple t = it.next();
- Long l = (Long)t.get(0);
- Long lSquare = (Long)t.get(1);
- Long count = (Long)t.get(2);
-
- // we count nulls in var as contributing 0
- // a departure from SQL for performance of
- // COUNT() which implemented by just inspecting
- // size of the bag
- if(l == null) {
- l = (long)0;
- lSquare = (long)0;
- } else {
- sawNonNull = true;
- }
- sum += l;
- sumSquare += lSquare;
- totalCount += count;
- }
- if(sawNonNull) {
- output.set(0, new Long(sum));
- output.set(1, new Long(sumSquare));
- } else {
- output.set(0, null);
- output.set(1, null);
- }
- output.set(2, Long.valueOf(totalCount));
- return output;
- }
-
- static protected long count(Tuple input) throws ExecException {
- DataBag values = (DataBag)input.get(0);
- long cnt = 0;
- Iterator<Tuple> it = values.iterator();
- while (it.hasNext()){
- Tuple t = (Tuple)it.next();
- if (t != null && t.size() > 0 && t.get(0) != null)
- cnt ++;
- }
-
- return cnt;
- }
-
- static protected Long sum(Tuple input) throws ExecException, IOException {
- DataBag values = (DataBag)input.get(0);
-
- // if we were handed an empty bag, return NULL
- if(values.size() == 0) {
- return null;
- }
-
- long sum = 0;
- boolean sawNonNull = false;
- for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
- Tuple t = it.next();
- try{
- Long l = (Long)t.get(0);
- if (l == null) continue;
- sawNonNull = true;
- sum += l;
- }catch(RuntimeException exp) {
- int errCode = 2103;
- String msg = "Problem while computing sum of values.";
- throw new ExecException(msg, errCode, PigException.BUG, exp);
- }
- }
-
- if(sawNonNull) {
- return new Long(sum);
- } else {
- return null;
- }
- }
-
- static protected Long sumSquare(Tuple input) throws ExecException, IOException {
- DataBag values = (DataBag)input.get(0);
-
- // if we were handed an empty bag, return NULL
- if(values.size() == 0) {
- return null;
- }
-
- long sumSquare = 0;
- boolean sawNonNull = false;
- for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
- Tuple t = it.next();
- try{
- Long l = (Long)t.get(0);
- if (l == null) continue;
- sawNonNull = true;
- sumSquare += l*l;
- }catch(RuntimeException exp) {
- int errCode = 2103;
- String msg = "Problem while computing sum of squared values.";
- throw new ExecException(msg, errCode, PigException.BUG, exp);
- }
- }
-
- if(sawNonNull) {
- return new Long(sumSquare);
- } else {
- return null;
- }
- }
-
- @Override
- public Schema outputSchema(Schema input) {
- return new Schema(new Schema.FieldSchema(null, DataType.DOUBLE));
- }
-
- /* Accumulator interface implementation */
- private Long intermediateSumSquare = null;
- private Long intermediateSum = null;
- private Long intermediateCount = null;
-
- @Override
- public void accumulate(Tuple b) throws IOException {
- try {
- Long sum = sum(b);
- if(sum == null) {
- return;
- }
-
- Long sumSquare = sumSquare(b);
- if(sumSquare == null) {
- return;
- }
-
- // set default values
- if (intermediateSum == null || intermediateCount == null) {
- intermediateSumSquare = (long) 0;
- intermediateSum = (long) 0;
- intermediateCount = (long) 0;
- }
-
- long count = (Long)count(b);
-
- if (count > 0) {
- intermediateCount += count;
- intermediateSum += sum;
- intermediateSumSquare += sumSquare;
- }
- } catch (ExecException ee) {
- throw ee;
- } catch (Exception e) {
- int errCode = 2106;
- String msg = "Error while computing variance in " + this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
- }
- }
-
- @Override
- public void cleanup() {
- intermediateSumSquare = null;
- intermediateSum = null;
- intermediateCount = null;
- }
-
- @Override
- public Double getValue() {
- Double var = null;
- if (intermediateCount != null && intermediateCount > 0) {
- Double avg = new Double((double)intermediateSum / intermediateCount);
- Double avgSquare = new Double((double)intermediateSumSquare / intermediateCount);
- var = avgSquare - avg*avg;
- }
- return var;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/MarkovPairs.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/MarkovPairs.java b/src/java/datafu/pig/stats/MarkovPairs.java
deleted file mode 100644
index d3d7632..0000000
--- a/src/java/datafu/pig/stats/MarkovPairs.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-
-import org.apache.pig.EvalFunc;
-import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
-
-
-/**
- * Accepts a bag of tuples, with user supplied ordering, and generates pairs that can be used for
- * a Markov chain analysis. For example, if you had {(1), (4), (7)}, using the default lookahead of 1, you
- * get the pairs {
- * ((1),(4)),
- * ((4),(7))}
- * A lookahead factor tells the UDF how many steps in to the future to include. so, for a,b,c with a lookahead
- * of 2, a would be paired with both b and c.
- * The results are ordered are returned as ordered by the caller.
-*/
-
-public class MarkovPairs extends EvalFunc<DataBag>
-{
- private static final BagFactory bagFactory = BagFactory.getInstance();
- private static final TupleFactory tupleFactory = TupleFactory.getInstance();
-
- private static long lookahead_steps;
-
- private final int SPILL_THRESHOLD = 1000000;
-
- public MarkovPairs()
- {
- MarkovPairs.lookahead_steps = 1;
- }
-
- public MarkovPairs(String lookahead_steps)
- {
- MarkovPairs.lookahead_steps = Integer.valueOf(lookahead_steps);
- }
-
- /* start and end are inclusive. This forms transition pairs */
- private void generatePairs(ArrayList<Tuple> input, int start, int end, DataBag outputBag)
- throws ExecException
- {
- int count = 0;
- for (int i = start; (i + 1)<= end; i++) {
- Tuple elem1 = input.get(i);
-
- lookahead:
- for (int j = i+1; j <= i + lookahead_steps; j++)
- {
- if (j > end) break lookahead;
- Tuple elem2 = input.get(j);
- if (count >= SPILL_THRESHOLD) {
- outputBag.spill();
- count = 0;
- }
- outputBag.add(tupleFactory.newTuple(Arrays.asList(elem1, elem2)));
- count ++;
- }
- }
- }
-
- @Override
- public DataBag exec(Tuple input)
- throws IOException
- {
- //things come in a tuple, in our case we have a bag (ordered views) passed. This is embedded in a length one tuple
-
- DataBag inputBag = (DataBag) input.get(0);
- ArrayList<Tuple> inputData = new ArrayList<Tuple>();
-
- for (Tuple tuple : inputBag) {
- inputData.add(tuple);
- }
-
- int inputSize = inputData.size();
-
- try {
- DataBag outputBag = bagFactory.newDefaultBag();
-
- int startPos = 0;
-
- int stopPos = inputSize - 1;
- generatePairs(inputData, startPos, stopPos, outputBag);
-
- // set startPos for the next bucket
- startPos = stopPos + 1;
- return outputBag;
- }
- catch (Exception e) {
- throw new IOException(e);
- }
- }
-
-
- @Override
- public Schema outputSchema(Schema input)
- {
- try {
- Schema tupleSchema = new Schema();
-
- FieldSchema fieldSchema = input.getField(0);
-
- if (fieldSchema.type != DataType.BAG)
- {
- throw new RuntimeException(String.format("Expected input schema to be BAG, but instead found %s",
- DataType.findTypeName(fieldSchema.type)));
- }
-
- FieldSchema fieldSchema2 = fieldSchema.schema.getField(0);
-
- tupleSchema.add(new Schema.FieldSchema("elem1", fieldSchema2.schema));
- tupleSchema.add(new Schema.FieldSchema("elem2", fieldSchema2.schema));
- return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
- tupleSchema,
- DataType.BAG));
- }
- catch (Exception e) {
- return null;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/Median.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/Median.java b/src/java/datafu/pig/stats/Median.java
deleted file mode 100644
index e33a84e..0000000
--- a/src/java/datafu/pig/stats/Median.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats;
-
-/**
- * Computes the {@link <a href="http://en.wikipedia.org/wiki/Median" target="_blank">median</a>}
- * for a <b>sorted</b> input bag, using type R-2 estimation. This is a convenience wrapper around Quantile.
- *
- * <p>
- * N.B., all the data is pushed to a single reducer per key, so make sure some partitioning is
- * done (e.g., group by 'day') if the data is too large. That is, this isn't distributed median.
- * </p>
- *
- * @see Quantile
- */
-public class Median extends Quantile
-{
- public Median()
- {
- super("0.5");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/src/java/datafu/pig/stats/Quantile.java
----------------------------------------------------------------------
diff --git a/src/java/datafu/pig/stats/Quantile.java b/src/java/datafu/pig/stats/Quantile.java
deleted file mode 100644
index 6fd42d3..0000000
--- a/src/java/datafu/pig/stats/Quantile.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.pig.stats;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataType;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
-
-import datafu.pig.util.SimpleEvalFunc;
-
-/**
- * Computes {@link <a href="http://en.wikipedia.org/wiki/Quantile" target="_blank">quantiles</a>}
- * for a <b>sorted</b> input bag, using type R-2 estimation.
- *
- * <p>
- * N.B., all the data is pushed to a single reducer per key, so make sure some partitioning is
- * done (e.g., group by 'day') if the data is too large. That is, this isn't distributed quantiles.
- * </p>
- *
- * <p>
- * Note that unlike datafu's StreamingQuantile algorithm, this implementation gives
- * <b>exact</b> quantiles. But, it requires that the input bag to be sorted. Quantile must spill to
- * disk when the input data is too large to fit in memory, which will contribute to longer runtimes.
- * Because StreamingQuantile implements accumulate it can be much more efficient than Quantile for
- * large input bags which do not fit well in memory.
- * </p>
- *
- * <p>The constructor takes a single integer argument that specifies the number of evenly-spaced
- * quantiles to compute, e.g.,</p>
- *
- * <ul>
- * <li>Quantile('3') yields the min, the median, and the max
- * <li>Quantile('5') yields the min, the 25th, 50th, 75th percentiles, and the max
- * <li>Quantile('101') yields the min, the max, and all 99 percentiles.
- * </ul>
- *
- * <p>Alternatively the constructor can take the explicit list of quantiles to compute, e.g.</p>
- *
- * <ul>
- * <li>Quantile('0.0','0.5','1.0') yields the min, the median, and the max
- * <li>Quantile('0.0','0.25','0.5','0.75','1.0') yields the min, the 25th, 50th, 75th percentiles, and the max
- * </ul>
- *
- * <p>The list of quantiles need not span the entire range from 0.0 to 1.0, nor do they need to be evenly spaced, e.g.</p>
- *
- * <ul>
- * <li>Quantile('0.5','0.90','0.95','0.99') yields the median, the 90th, 95th, and the 99th percentiles
- * <li>Quantile('0.0013','0.0228','0.1587','0.5','0.8413','0.9772','0.9987') yields the 0.13th, 2.28th, 15.87th, 50th, 84.13th, 97.72nd, and 99.87th percentiles
- * </ul>
- *
- * <p>
- * Example:
- * <pre>
- * {@code
- *
- * define Quantile datafu.pig.stats.Quantile('0.0','0.5','1.0');
-
- * -- input: 9,10,2,3,5,8,1,4,6,7
- * input = LOAD 'input' AS (val:int);
- *
- * grouped = GROUP input ALL;
- *
- * -- produces: (1,5.5,10)
- * quantiles = FOREACH grouped {
- * sorted = ORDER input BY val;
- * GENERATE Quantile(sorted);
- * }
- * }</pre></p>
- *
- * @see Median
- * @see StreamingQuantile
- */
-public class Quantile extends SimpleEvalFunc<Tuple>
-{
- List<Double> quantiles;
-
- private static class Pair<T1,T2>
- {
- public T1 first;
- public T2 second;
-
- public Pair(T1 first, T2 second) {
- this.first = first;
- this.second = second;
- }
- }
-
- // For the output schema, label the quantiles 0, 1, 2, ... n
- // Otherwise label the quantiles based on the quantile value.
- // e.g. 50% quantile 0.5 will be labeled as 0_5
- private boolean ordinalOutputSchema;
-
- public Quantile(String... k)
- {
- this.quantiles = QuantileUtil.getQuantilesFromParams(k);
-
- if (k.length == 1 && Double.parseDouble(k[0]) > 1.0)
- {
- this.ordinalOutputSchema = true;
- this.quantiles = QuantileUtil.getQuantilesFromParams(k);
- }
- else
- {
- this.quantiles = QuantileUtil.getQuantilesFromParams(k);
- }
- }
-
- private static Pair<Long, Long> getIndexes(double k, long N)
- {
- double h = N*k + 0.5;
- long i1 = Math.min(Math.max(1, (long)Math.ceil(h - 0.5)), N);
- long i2 = Math.min(Math.max(1, (long)Math.floor(h + 0.5)), N);
-
- return new Pair<Long, Long>(i1, i2);
- }
-
- public Tuple call(DataBag bag) throws IOException
- {
- if (bag == null || bag.size() == 0)
- return null;
-
- Map<Long, Double> d = new HashMap<Long, Double>();
- long N = bag.size(), max_id = 1;
-
- for (double k : this.quantiles) {
- Pair<Long, Long> idx = getIndexes(k, N);
-
- d.put(idx.first, null);
- d.put(idx.second, null);
- max_id = Math.max(max_id, idx.second);
- }
-
- long i = 1;
- for (Tuple t : bag) {
- if (i > max_id)
- break;
-
- if (d.containsKey(i)) {
- Object o = t.get(0);
- if (!(o instanceof Number))
- throw new IllegalStateException("bag must have numerical values (and be non-null)");
- d.put(i, ((Number) o).doubleValue());
- }
- i++;
- }
-
- Tuple t = TupleFactory.getInstance().newTuple(this.quantiles.size());
- int j = 0;
- for (double k : this.quantiles) {
- Pair<Long, Long> p = getIndexes(k, N);
- double quantile = (d.get(p.first) + d.get(p.second)) / 2;
- t.set(j, quantile);
- j++;
- }
- return t;
- }
-
- @Override
- public Schema outputSchema(Schema input)
- {
- Schema tupleSchema = new Schema();
- if (ordinalOutputSchema)
- {
- for (int i = 0; i < this.quantiles.size(); i++)
- {
- tupleSchema.add(new Schema.FieldSchema("quantile_" + i, DataType.DOUBLE));
- }
- }
- else
- {
- for (Double x : this.quantiles)
- tupleSchema.add(new Schema.FieldSchema("quantile_" + x.toString().replace(".", "_"), DataType.DOUBLE));
- }
-
- try {
- return new Schema(new FieldSchema(null, tupleSchema, DataType.TUPLE));
- } catch(FrontendException e) {
- throw new RuntimeException(e);
- }
- }
-}
-
[04/19] DATAFU-27 Migrate build system to Gradle
Posted by wv...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/PigTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/PigTests.java b/test/pig/datafu/test/pig/PigTests.java
deleted file mode 100644
index 9b4eddd..0000000
--- a/test/pig/datafu/test/pig/PigTests.java
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig;
-
-import static org.testng.Assert.*;
-
-import java.io.BufferedInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileWriter;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.regex.Pattern;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.metrics.jvm.JvmMetrics;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.pigunit.PigTest;
-import org.apache.pig.tools.parameters.ParseException;
-
-public abstract class PigTests
-{
- @org.testng.annotations.BeforeClass
- public void beforeClass()
- {
- // TODO make it configurable whether this happens, for travis-ci we can't spam the logs so much,
- // however otherwise it is useful to see the errors
- Logger.getRootLogger().removeAllAppenders();
- Logger.getLogger(JvmMetrics.class).setLevel(Level.OFF);
- }
-
- @org.testng.annotations.BeforeMethod
- public void beforeMethod(Method method)
- {
- System.out.println("\n*** Running " + method.getName() + " ***");
- }
-
- protected String[] getDefaultArgs()
- {
- String[] args = {
- "JAR_PATH=" + getJarPath()
- };
- return args;
- }
-
- protected List<String> getDefaultArgsAsList()
- {
- String[] args = getDefaultArgs();
- List<String> argsList = new ArrayList<String>(args.length);
- for (String arg : args)
- {
- argsList.add(arg);
- }
- return argsList;
- }
-
- protected PigTest createPigTestFromString(String str, String... args) throws IOException
- {
- return createPigTest(str.split("\n"),args);
- }
-
- protected PigTest createPigTest(String[] lines, String... args) throws IOException
- {
- // append args to list of default args
- List<String> theArgs = getDefaultArgsAsList();
- for (String arg : args)
- {
- theArgs.add(arg);
- }
-
- for (String arg : theArgs)
- {
- String[] parts = arg.split("=",2);
- if (parts.length == 2)
- {
- for (int i=0; i<lines.length; i++)
- {
- lines[i] = lines[i].replaceAll(Pattern.quote("$" + parts[0]), parts[1]);
- }
- }
- }
-
- return new PigTest(lines);
- }
-
- protected PigTest createPigTest(String scriptPath, String... args) throws IOException
- {
- return createPigTest(getLinesFromFile(scriptPath), args);
- }
-
- protected String getJarPath()
- {
- String jarDir = null;
-
- if (System.getProperty("datafu.jar.dir") != null)
- {
- jarDir = System.getProperty("datafu.jar.dir");
- }
- else
- {
- jarDir = new File(System.getProperty("user.dir"), "dist").getAbsolutePath();
- }
-
- File userDir = new File(jarDir);
-
- String[] files = userDir.list(new FilenameFilter() {
-
- @Override
- public boolean accept(File dir, String name)
- {
- return name.endsWith(".jar") && !name.contains("sources") && !name.contains("javadoc");
- }
-
- });
-
- if (files == null || files.length == 0)
- {
- throw new RuntimeException("Could not find JAR file");
- }
- else if (files.length > 1)
- {
- StringBuilder sb = new StringBuilder();
- for (String file : files)
- {
- sb.append(file);
- sb.append(",");
- }
- throw new RuntimeException("Found more JAR files than expected: " + sb.substring(0, sb.length()-1));
- }
-
- return userDir.getAbsolutePath() + "/" + files[0];
- }
-
- protected List<Tuple> getLinesForAlias(PigTest test, String alias) throws IOException, ParseException
- {
- return getLinesForAlias(test,alias,true);
- }
-
- protected List<Tuple> getLinesForAlias(PigTest test, String alias, boolean logValues) throws IOException, ParseException
- {
- Iterator<Tuple> tuplesIterator = test.getAlias(alias);
- List<Tuple> tuples = new ArrayList<Tuple>();
- if (logValues)
- {
- System.out.println(String.format("Values for %s: ", alias));
- }
- while (tuplesIterator.hasNext())
- {
- Tuple tuple = tuplesIterator.next();
- if (logValues)
- {
- System.out.println(tuple.toString());
- }
- tuples.add(tuple);
- }
- return tuples;
- }
-
- protected void writeLinesToFile(String fileName, String... lines) throws IOException
- {
- File inputFile = deleteIfExists(getFile(fileName));
- writeLinesToFile(inputFile, lines);
- }
-
- protected void writeLinesToFile(File file, String[] lines) throws IOException
- {
- FileWriter writer = new FileWriter(file);
- for (String line : lines)
- {
- writer.write(line + "\n");
- }
- writer.close();
- }
-
- protected void assertOutput(PigTest test, String alias, String... expected) throws IOException, ParseException
- {
- List<Tuple> tuples = getLinesForAlias(test, alias);
- assertEquals(expected.length, tuples.size());
- int i=0;
- for (String e : expected)
- {
- assertEquals(tuples.get(i++).toString(), e);
- }
- }
-
- protected File deleteIfExists(File file)
- {
- if (file.exists())
- {
- file.delete();
- }
- return file;
- }
-
- protected File getFile(String fileName)
- {
- return new File(System.getProperty("user.dir"), fileName).getAbsoluteFile();
- }
-
- /**
- * Gets the lines from a given file.
- *
- * @param relativeFilePath The path relative to the datafu-tests project.
- * @return The lines from the file
- * @throws IOException
- */
- protected String[] getLinesFromFile(String relativeFilePath) throws IOException
- {
- // assume that the working directory is the datafu-tests project
- File file = new File(System.getProperty("user.dir"), relativeFilePath).getAbsoluteFile();
- BufferedInputStream content = new BufferedInputStream(new FileInputStream(file));
- Object[] lines = IOUtils.readLines(content).toArray();
- String[] result = new String[lines.length];
- for (int i=0; i<lines.length; i++)
- {
- result[i] = (String)lines[i];
- }
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/bags/BagTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/bags/BagTests.java b/test/pig/datafu/test/pig/bags/BagTests.java
deleted file mode 100644
index 80bb0cc..0000000
--- a/test/pig/datafu/test/pig/bags/BagTests.java
+++ /dev/null
@@ -1,1254 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.bags;
-
-import static org.testng.Assert.assertEquals;
-
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Set;
-
-import junit.framework.Assert;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.pig.bags.CountEach;
-import datafu.pig.bags.DistinctBy;
-import datafu.pig.bags.Enumerate;
-import datafu.test.pig.PigTests;
-
-
-public class BagTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- define NullToEmptyBag datafu.pig.bags.NullToEmptyBag();
-
- data = LOAD 'input' AS (B: bag {T: tuple(v:INT)});
-
- dump data;
-
- data2 = FOREACH data GENERATE NullToEmptyBag(B) as P;
-
- dump data2;
-
- STORE data2 INTO 'output';
- */
- @Multiline
- private String nullToEmptyBag;
-
- @Test
- public void nullToEmptyBagTest() throws Exception
- {
- PigTest test = createPigTestFromString(nullToEmptyBag);
-
- writeLinesToFile("input",
- "({(1),(2),(3),(4),(5)})",
- "()",
- "{(4),(5)})");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "({(1),(2),(3),(4),(5)})",
- "({})",
- "({(4),(5)})");
- }
-
- /**
- register $JAR_PATH
-
- define EmptyBagToNull datafu.pig.bags.EmptyBagToNull();
-
- data = LOAD 'input' AS (B: bag {T: tuple(v:INT)});
-
- dump data;
-
- data2 = FOREACH data GENERATE EmptyBagToNull(B) as P;
-
- dump data2;
-
- STORE data2 INTO 'output';
- */
- @Multiline
- private String emptyBagToNullTest;
-
- @Test
- public void emptyBagToNullTest() throws Exception
- {
- PigTest test = createPigTestFromString(emptyBagToNullTest);
-
- writeLinesToFile("input",
- "({(1),(2),(3),(4),(5)})",
- "()",
- "({})",
- "{(4),(5)})");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "({(1),(2),(3),(4),(5)})",
- "()",
- "()",
- "({(4),(5)})");
- }
-
- /**
- register $JAR_PATH
-
- define EmptyBagToNullFields datafu.pig.bags.EmptyBagToNullFields();
-
- data = LOAD 'input' AS (B: bag {T: tuple(v1:INT,v2:INT)});
-
- dump data;
-
- data2 = FOREACH data GENERATE EmptyBagToNullFields(B) as P;
-
- dump data2;
-
- STORE data2 INTO 'output';
- */
- @Multiline
- private String emptyBagToNullFieldsTest;
-
- @Test
- public void emptyBagToNullFieldsTest() throws Exception
- {
- PigTest test = createPigTestFromString(emptyBagToNullFieldsTest);
-
- writeLinesToFile("input",
- "({(1,1),(2,2),(3,3),(4,4),(5,5)})",
- "({})",
- "{(4,4),(5,5)})");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "({(1,1),(2,2),(3,3),(4,4),(5,5)})",
- "({(,)})",
- "({(4,4),(5,5)})");
- }
-
- /**
- register $JAR_PATH
-
- define AppendToBag datafu.pig.bags.AppendToBag();
-
- data = LOAD 'input' AS (key:INT, B: bag{T: tuple(v:INT)}, T: tuple(v:INT));
-
- data2 = FOREACH data GENERATE key, AppendToBag(B,T) as B;
-
- STORE data2 INTO 'output';
-
- */
- @Multiline
- private String appendToBagTest;
-
- @Test
- public void appendToBagTest() throws Exception
- {
- PigTest test = createPigTestFromString(appendToBagTest);
-
- writeLinesToFile("input",
- "1\t{(1),(2),(3)}\t(4)",
- "2\t{(10),(20),(30),(40),(50)}\t(60)");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "(1,{(1),(2),(3),(4)})",
- "(2,{(10),(20),(30),(40),(50),(60)})");
- }
-
- /**
- register $JAR_PATH
-
- define FirstTupleFromBag datafu.pig.bags.FirstTupleFromBag();
-
- data = LOAD 'input' AS (key:INT, B: bag{T: tuple(v:INT)});
-
- data2 = FOREACH data GENERATE key, FirstTupleFromBag(B, null) as B;
-
- STORE data2 INTO 'output';
-
- */
- @Multiline
- private String firstTupleFromBagTest;
-
- @Test
- public void firstTupleFromBagTest() throws Exception
- {
- PigTest test = createPigTestFromString(firstTupleFromBagTest);
-
- writeLinesToFile("input", "1\t{(4),(9),(16)}");
-
- test.runScript();
-
- assertOutput(test, "data2", "(1,(4))");
- }
-
- /**
- register $JAR_PATH
-
- define PrependToBag datafu.pig.bags.PrependToBag();
-
- data = LOAD 'input' AS (key:INT, B: bag{T: tuple(v:INT)}, T: tuple(v:INT));
-
- data2 = FOREACH data GENERATE key, PrependToBag(B,T) as B;
-
- STORE data2 INTO 'output';
-
- */
- @Multiline
- private String prependToBagTest;
-
- @Test
- public void prependToBagTest() throws Exception
- {
- PigTest test = createPigTestFromString(prependToBagTest);
-
- writeLinesToFile("input",
- "1\t{(1),(2),(3)}\t(4)",
- "2\t{(10),(20),(30),(40),(50)}\t(60)");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "(1,{(4),(1),(2),(3)})",
- "(2,{(60),(10),(20),(30),(40),(50)})");
- }
-
- /**
- register $JAR_PATH
-
- define BagConcat datafu.pig.bags.BagConcat();
-
- data = LOAD 'input' AS (A: bag{T: tuple(v:INT)}, B: bag{T: tuple(v:INT)}, C: bag{T: tuple(v:INT)});
-
- describe data;
-
- data2 = FOREACH data GENERATE BagConcat(A,B,C);
-
- describe data2;
-
- STORE data2 INTO 'output';
- */
- @Multiline
- private String bagConcatTest;
-
- @Test
- public void bagConcatTest() throws Exception
- {
- PigTest test = createPigTestFromString(bagConcatTest);
-
- writeLinesToFile("input",
- "({(1),(2),(3)}\t{(3),(5),(6)}\t{(10),(13)})",
- "({(2),(3),(4)}\t{(5),(5)}\t{(20)})");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "({(1),(2),(3),(3),(5),(6),(10),(13)})",
- "({(2),(3),(4),(5),(5),(20)})");
- }
-
- /**
- register $JAR_PATH
-
- define UnorderedPairs datafu.pig.bags.UnorderedPairs();
-
- data = LOAD 'input' AS (B: bag {T: tuple(v:INT)});
-
- data2 = FOREACH data GENERATE UnorderedPairs(B) as P;
-
- data3 = FOREACH data2 GENERATE FLATTEN(P);
-
- data4 = FOREACH data3 GENERATE FLATTEN(elem1), FLATTEN(elem2);
-
- data5 = ORDER data4 BY $0, $1;
-
- STORE data5 INTO 'output';
-
-
- */
- @Multiline
- private String unorderedPairsTest;
-
- @Test
- public void unorderedPairsTest() throws Exception
- {
- PigTest test = createPigTestFromString(unorderedPairsTest);
-
- String[] input = {
- "{(1),(2),(3),(4),(5)}"
- };
-
- String[] output = {
- "(1,2)",
- "(1,3)",
- "(1,4)",
- "(1,5)",
- "(2,3)",
- "(2,4)",
- "(2,5)",
- "(3,4)",
- "(3,5)",
- "(4,5)"
- };
-
- test.assertOutput("data",input,"data4",output);
- }
-
- /**
- register $JAR_PATH
-
- define UnorderedPairs datafu.pig.bags.UnorderedPairs();
-
- data = LOAD 'input' AS (A:int, B: bag {T: tuple(v:INT)});
-
- data2 = FOREACH data GENERATE A, UnorderedPairs(B) as P;
-
- data3 = FOREACH data2 GENERATE A, FLATTEN(P);
-
- STORE data3 INTO 'output';
-
- */
- @Multiline
- private String unorderedPairsTest2;
-
- @Test
- public void unorderedPairsTest2() throws Exception
- {
- PigTest test = createPigTestFromString(unorderedPairsTest2);
-
- this.writeLinesToFile("input", "1\t{(1),(2),(3),(4),(5)}");
-
- test.runScript();
- this.getLinesForAlias(test, "data3");
-
- this.assertOutput(test, "data3",
- "(1,(1),(2))",
- "(1,(1),(3))",
- "(1,(1),(4))",
- "(1,(1),(5))",
- "(1,(2),(3))",
- "(1,(2),(4))",
- "(1,(2),(5))",
- "(1,(3),(4))",
- "(1,(3),(5))",
- "(1,(4),(5))");
- }
-
- /**
- register $JAR_PATH
-
- define BagSplit datafu.pig.bags.BagSplit();
-
- data = LOAD 'input' AS (B:bag{T:tuple(val1:INT,val2:INT)});
-
- data2 = FOREACH data GENERATE BagSplit($MAX,B);
- --describe data2;
-
- data3 = FOREACH data2 GENERATE FLATTEN($0);
-
- --describe data3
-
- STORE data3 INTO 'output';
-
- */
- @Multiline
- private String bagSplitTest;
-
- @Test
- public void bagSplitTest() throws Exception
- {
- PigTest test = createPigTestFromString(bagSplitTest,
- "MAX=5");
-
- writeLinesToFile("input",
- "{(1,11),(2,22),(3,33),(4,44),(5,55),(6,66),(7,77),(8,88),(9,99),(10,1010),(11,1111),(12,1212)}");
-
- test.runScript();
-
- assertOutput(test, "data3",
- "({(1,11),(2,22),(3,33),(4,44),(5,55)})",
- "({(6,66),(7,77),(8,88),(9,99),(10,1010)})",
- "({(11,1111),(12,1212)})");
- }
-
- /**
- register $JAR_PATH
-
- define BagSplit datafu.pig.bags.BagSplit('true');
-
- data = LOAD 'input' AS (B:bag{T:tuple(val1:INT,val2:INT)});
-
- data2 = FOREACH data GENERATE BagSplit($MAX,B);
-
- data3 = FOREACH data2 GENERATE FLATTEN($0);
-
- STORE data3 INTO 'output';
- */
- @Multiline
- private String bagSplitWithBagNumTest;
-
- @Test
- public void bagSplitWithBagNumTest() throws Exception
- {
- PigTest test = createPigTestFromString(bagSplitWithBagNumTest,
- "MAX=10");
-
- writeLinesToFile("input",
- "{(1,11),(2,22),(3,33),(4,44),(5,55),(6,66),(7,77),(8,88),(9,99),(10,1010),(11,1111),(12,1212)}");
-
- test.runScript();
-
- assertOutput(test, "data3",
- "({(1,11),(2,22),(3,33),(4,44),(5,55),(6,66),(7,77),(8,88),(9,99),(10,1010)},0)",
- "({(11,1111),(12,1212)},1)");
- }
-
- /**
- register $JAR_PATH
-
- define Enumerate datafu.pig.bags.ReverseEnumerate('1');
-
- data = LOAD 'input' AS (data: bag {T: tuple(v1:INT,B: bag{T: tuple(v2:INT)})});
-
- data2 = FOREACH data GENERATE Enumerate(data);
- --describe data2;
-
- data3 = FOREACH data2 GENERATE FLATTEN($0);
- --describe data3;
-
- data4 = FOREACH data3 GENERATE $0 as v1, $1 as B, $2 as i;
- --describe data4;
-
- STORE data4 INTO 'output';
-
- */
- @Multiline
- private String enumerateWithReverseTest;
-
- @Test
- public void enumerateWithReverseTest() throws Exception
- {
- PigTest test = createPigTestFromString(enumerateWithReverseTest);
-
- writeLinesToFile("input",
- "({(10,{(1),(2),(3)}),(20,{(4),(5),(6)}),(30,{(7),(8)}),(40,{(9),(10),(11)}),(50,{(12),(13),(14),(15)})})");
-
- test.runScript();
-
- assertOutput(test, "data4",
- "(10,{(1),(2),(3)},5)",
- "(20,{(4),(5),(6)},4)",
- "(30,{(7),(8)},3)",
- "(40,{(9),(10),(11)},2)",
- "(50,{(12),(13),(14),(15)},1)");
- }
-
- /**
- register $JAR_PATH
-
- define Enumerate datafu.pig.bags.Enumerate('1');
-
- data = LOAD 'input' AS (data: bag {T: tuple(v1:INT,B: bag{T: tuple(v2:INT)})});
-
- data2 = FOREACH data GENERATE Enumerate(data);
- --describe data2;
-
- data3 = FOREACH data2 GENERATE FLATTEN($0);
- --describe data3;
-
- data4 = FOREACH data3 GENERATE $0 as v1, $1 as B, $2 as i;
- --describe data4;
-
- STORE data4 INTO 'output';
-
- */
- @Multiline
- private String enumerateWithStartTest;
-
- @Test
- public void enumerateWithStartTest() throws Exception
- {
- PigTest test = createPigTestFromString(enumerateWithStartTest);
-
- writeLinesToFile("input",
- "({(10,{(1),(2),(3)}),(20,{(4),(5),(6)}),(30,{(7),(8)}),(40,{(9),(10),(11)}),(50,{(12),(13),(14),(15)})})");
-
- test.runScript();
-
- assertOutput(test, "data4",
- "(10,{(1),(2),(3)},1)",
- "(20,{(4),(5),(6)},2)",
- "(30,{(7),(8)},3)",
- "(40,{(9),(10),(11)},4)",
- "(50,{(12),(13),(14),(15)},5)");
- }
-
- /**
- register $JAR_PATH
-
- define Enumerate datafu.pig.bags.Enumerate();
-
- data = LOAD 'input' AS (data: bag {T: tuple(v1:INT,B: bag{T: tuple(v2:INT)})});
-
- data2 = FOREACH data GENERATE Enumerate(data);
- --describe data2;
-
- data3 = FOREACH data2 GENERATE FLATTEN($0);
- --describe data3;
-
- data4 = FOREACH data3 GENERATE $0 as v1, $1 as B, $2 as i;
- --describe data4;
-
- STORE data4 INTO 'output';
-
- */
- @Multiline
- private String enumerateTest;
-
- @Test
- public void enumerateTest() throws Exception
- {
- PigTest test = createPigTestFromString(enumerateTest);
-
- writeLinesToFile("input",
- "({(10,{(1),(2),(3)}),(20,{(4),(5),(6)}),(30,{(7),(8)}),(40,{(9),(10),(11)}),(50,{(12),(13),(14),(15)})})");
-
- test.runScript();
-
- assertOutput(test, "data4",
- "(10,{(1),(2),(3)},0)",
- "(20,{(4),(5),(6)},1)",
- "(30,{(7),(8)},2)",
- "(40,{(9),(10),(11)},3)",
- "(50,{(12),(13),(14),(15)},4)");
- }
-
- @Test
- public void enumerateTest2() throws Exception
- {
- PigTest test = createPigTestFromString(enumerateTest);
-
- writeLinesToFile("input",
- "({(10,{(1),(2),(3)}),(20,{(4),(5),(6)}),(30,{(7),(8)}),(40,{(9),(10),(11)}),(50,{(12),(13),(14),(15)})})",
- "({(11,{(11),(12),(13),(14)}),(21,{(15),(16),(17),(18)}),(31,{(19),(20)}),(41,{(21),(22),(23),(24)}),(51,{(25),(26),(27)})})");
-
- test.runScript();
-
- assertOutput(test, "data4",
- "(10,{(1),(2),(3)},0)",
- "(20,{(4),(5),(6)},1)",
- "(30,{(7),(8)},2)",
- "(40,{(9),(10),(11)},3)",
- "(50,{(12),(13),(14),(15)},4)",
- "(11,{(11),(12),(13),(14)},0)",
- "(21,{(15),(16),(17),(18)},1)",
- "(31,{(19),(20)},2)",
- "(41,{(21),(22),(23),(24)},3)",
- "(51,{(25),(26),(27)},4)");
- }
-
- /*
- * Testing "Accumulator" part of Enumeration by manually invoking accumulate(), getValue() and cleanup()
- */
- @Test
- public void enumerateAccumulatorTest() throws Exception
- {
- Enumerate enumerate = new Enumerate();
-
- Tuple tuple1 = TupleFactory.getInstance().newTuple(1);
- tuple1.set(0, 10);
-
- Tuple tuple2 = TupleFactory.getInstance().newTuple(1);
- tuple2.set(0, 20);
-
- Tuple tuple3 = TupleFactory.getInstance().newTuple(1);
- tuple3.set(0, 30);
-
- Tuple tuple4 = TupleFactory.getInstance().newTuple(1);
- tuple4.set(0, 40);
-
- Tuple tuple5 = TupleFactory.getInstance().newTuple(1);
- tuple5.set(0, 50);
-
- DataBag bag1 = BagFactory.getInstance().newDefaultBag();
- bag1.add(tuple1);
- bag1.add(tuple2);
- bag1.add(tuple3);
-
- DataBag bag2 = BagFactory.getInstance().newDefaultBag();
- bag2.add(tuple4);
- bag2.add(tuple5);
-
- Tuple inputTuple1 = TupleFactory.getInstance().newTuple(1);
- inputTuple1.set(0,bag1);
-
- Tuple inputTuple2 = TupleFactory.getInstance().newTuple(1);
- inputTuple2.set(0,bag2);
-
- enumerate.accumulate(inputTuple1);
- enumerate.accumulate(inputTuple2);
- assertEquals(enumerate.getValue().toString(), "{(10,0),(20,1),(30,2),(40,3),(50,4)}");
-
- // Testing that cleanup code is correct by calling cleanup() and passing inputs back to Enumerate instance
- enumerate.cleanup();
- enumerate.accumulate(inputTuple1);
- enumerate.accumulate(inputTuple2);
- assertEquals(enumerate.getValue().toString(), "{(10,0),(20,1),(30,2),(40,3),(50,4)}");
- }
-
- /**
- register $JAR_PATH
-
- define BagSplit datafu.pig.bags.BagSplit();
- define Enumerate datafu.pig.bags.Enumerate('1');
-
- data = LOAD 'input' AS (data: bag {T: tuple(name:CHARARRAY, score:double)});
-
- data2 = FOREACH data GENERATE BagSplit(3,data) as the_bags;
-
- --describe data2
-
- data3 = FOREACH data2 GENERATE Enumerate(the_bags) as enumerated_bags;
-
- --describe data3
-
- data4 = FOREACH data3 GENERATE FLATTEN(enumerated_bags) as (data,i);
-
- --describe data4
-
- data5 = FOREACH data4 GENERATE data as the_data, i as the_key;
-
- --describe data5
-
- data_out = FOREACH data5 GENERATE FLATTEN(the_data), the_key;
-
- --describe data_out
- */
- @Multiline
- private String comprehensiveBagSplitAndEnumerate;
-
- @Test
- public void comprehensiveBagSplitAndEnumerate() throws Exception
- {
- PigTest test = createPigTestFromString(comprehensiveBagSplitAndEnumerate);
-
- writeLinesToFile("input",
- "({(A,1.0),(B,2.0),(C,3.0),(D,4.0),(E,5.0)})");
-
- test.runScript();
-
- assertOutput(test, "data_out",
- // bag #1
- "(A,1.0,1)",
- "(B,2.0,1)",
- "(C,3.0,1)",
- // bag #2
- "(D,4.0,2)",
- "(E,5.0,2)");
- }
-
- /**
- register $JAR_PATH
-
- define DistinctBy datafu.pig.bags.DistinctBy('0');
-
- data = LOAD 'input' AS (data: bag {T: tuple(a:CHARARRAY, b:INT, c:INT)});
-
- data2 = FOREACH data GENERATE DistinctBy(data);
-
- --describe data2;
-
- STORE data2 INTO 'output';
-
- */
- @Multiline
- private String distinctByTest;
-
- @Test
- public void distinctByTest() throws Exception
- {
- PigTest test = createPigTestFromString(distinctByTest);
-
- writeLinesToFile("input",
- "({(Z,1,0),(A,1,0),(A,1,0),(B,2,0),(B,22,1),(C,3,0),(D,4,0),(E,5,0)})",
- "({(A,10,2),(M,50,3),(A,34,49), (A,24,42), (Z,49,22),(B,1,1)},(B,2,2))");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "({(Z,1,0),(A,1,0),(B,2,0),(C,3,0),(D,4,0),(E,5,0)})",
- "({(A,10,2),(M,50,3),(Z,49,22),(B,1,1)})");
- }
-
- /**
- register $JAR_PATH
-
- define DistinctBy datafu.pig.bags.DistinctBy('1', '2');
-
- data = LOAD 'input' AS (data: bag {T: tuple(a:CHARARRAY, b:map[INT], c:bag{t: tuple(c0:CHARARRAY, c1:INT)})});
-
- data2 = FOREACH data GENERATE DistinctBy(data);
-
- --describe data2;
-
- STORE data2 INTO 'output';
-
- */
- @Multiline
- private String distinctByMultiComplexFieldTest;
-
- @Test
- public void distinctByMultiComplexFieldTest() throws Exception
- {
- PigTest test = createPigTestFromString(distinctByMultiComplexFieldTest);
-
- writeLinesToFile("input",
- "({(a-b,[a#0,b#1],{(a-b,0),(a-b,1)}),(a-c,[b#1,a#0],{(a-b,0),(a-b,1)}),(a-d,[a#1,b#0],{(a-b,1),(a-b,2)})})");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "({(a-b,[b#1,a#0],{(a-b,0),(a-b,1)}),(a-d,[b#0,a#1],{(a-b,1),(a-b,2)})})");
- }
-
- /**
- register $JAR_PATH
-
- define DistinctBy datafu.pig.bags.DistinctBy('1');
-
- data = LOAD 'input' AS (data: bag {T: tuple(a:CHARARRAY, b:CHARARRAY)});
-
- data2 = FOREACH data GENERATE DistinctBy(data);
-
- --describe data2;
-
- STORE data2 INTO 'output';
-
- */
- @Multiline
- private String distinctByDelimTest;
-
- @Test
- public void distinctByDelimTest() throws Exception
- {
- PigTest test = createPigTestFromString(distinctByDelimTest);
-
- writeLinesToFile("input",
- "({(a-b,c),(a-b,d)})");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "({(a-b,c),(a-b,d)})");
- }
-
- @Test
- public void distinctByExecTest() throws Exception
- {
- DistinctBy distinct = new DistinctBy("0");
-
- DataBag bag;
- Tuple input;
- Tuple data;
-
- bag = BagFactory.getInstance().newDefaultBag();
- input = TupleFactory.getInstance().newTuple(bag);
-
- data = TupleFactory.getInstance().newTuple(2);
- bag.add(data);
- data.set(0, 10);
- data.set(1, 20);
-
- data = TupleFactory.getInstance().newTuple(2);
- bag.add(data);
- data.set(0, 11);
- data.set(1, 50);
-
- data = TupleFactory.getInstance().newTuple(2);
- bag.add(data);
- data.set(0, 10);
- data.set(1, 22);
-
- data = TupleFactory.getInstance().newTuple(2);
- bag.add(data);
- data.set(0, 12);
- data.set(1, 40);
-
- data = TupleFactory.getInstance().newTuple(2);
- bag.add(data);
- data.set(0, 11);
- data.set(1, 50);
-
- data = TupleFactory.getInstance().newTuple(2);
- bag.add(data);
- data.set(0, 11);
- data.set(1, 51);
-
- DataBag result = distinct.exec(input);
-
- Assert.assertEquals(3, result.size());
-
- Iterator<Tuple> iter = result.iterator();
- Assert.assertEquals("(10,20)", iter.next().toString());
- Assert.assertEquals("(11,50)", iter.next().toString());
- Assert.assertEquals("(12,40)", iter.next().toString());
-
- // do it again to test cleanup
- bag = BagFactory.getInstance().newDefaultBag();
- input = TupleFactory.getInstance().newTuple(bag);
-
- data = TupleFactory.getInstance().newTuple(2);
- bag.add(data);
- data.set(0, 12);
- data.set(1, 42);
-
- data = TupleFactory.getInstance().newTuple(2);
- bag.add(data);
- data.set(0, 11);
- data.set(1, 51);
-
- data = TupleFactory.getInstance().newTuple(2);
- bag.add(data);
- data.set(0, 11);
- data.set(1, 50);
-
- result = distinct.exec(input);
-
- Assert.assertEquals(2, result.size());
-
- iter = result.iterator();
- Assert.assertEquals("(12,42)", iter.next().toString());
- Assert.assertEquals("(11,51)", iter.next().toString());
- }
-
- @Test
- public void distinctByAccumulateTest() throws Exception
- {
- DistinctBy distinct = new DistinctBy("0");
-
- DataBag bag;
- Tuple input;
- Tuple data;
-
- data = TupleFactory.getInstance().newTuple(2);
- bag = BagFactory.getInstance().newDefaultBag();
- bag.add(data);
- input = TupleFactory.getInstance().newTuple(bag);
- data.set(0, 10);
- data.set(1, 20);
- distinct.accumulate(input);
-
- data = TupleFactory.getInstance().newTuple(2);
- bag = BagFactory.getInstance().newDefaultBag();
- bag.add(data);
- input = TupleFactory.getInstance().newTuple(bag);
- data.set(0, 11);
- data.set(1, 50);
- distinct.accumulate(input);
-
- data = TupleFactory.getInstance().newTuple(2);
- bag = BagFactory.getInstance().newDefaultBag();
- bag.add(data);
- input = TupleFactory.getInstance().newTuple(bag);
- data.set(0, 10);
- data.set(1, 22);
- distinct.accumulate(input);
-
- data = TupleFactory.getInstance().newTuple(2);
- bag = BagFactory.getInstance().newDefaultBag();
- bag.add(data);
- input = TupleFactory.getInstance().newTuple(bag);
- data.set(0, 12);
- data.set(1, 40);
- distinct.accumulate(input);
-
- data = TupleFactory.getInstance().newTuple(2);
- bag = BagFactory.getInstance().newDefaultBag();
- bag.add(data);
- input = TupleFactory.getInstance().newTuple(bag);
- data.set(0, 11);
- data.set(1, 50);
- distinct.accumulate(input);
-
- data = TupleFactory.getInstance().newTuple(2);
- bag = BagFactory.getInstance().newDefaultBag();
- bag.add(data);
- input = TupleFactory.getInstance().newTuple(bag);
- data.set(0, 11);
- data.set(1, 51);
- distinct.accumulate(input);
-
- DataBag result = distinct.getValue();
-
- Assert.assertEquals(3, result.size());
-
- Iterator<Tuple> iter = result.iterator();
- Assert.assertEquals("(10,20)", iter.next().toString());
- Assert.assertEquals("(11,50)", iter.next().toString());
- Assert.assertEquals("(12,40)", iter.next().toString());
-
- // do it again to test cleanup
- distinct.cleanup();
-
- data = TupleFactory.getInstance().newTuple(2);
- bag = BagFactory.getInstance().newDefaultBag();
- bag.add(data);
- input = TupleFactory.getInstance().newTuple(bag);
- data.set(0, 12);
- data.set(1, 42);
- distinct.accumulate(input);
-
- data = TupleFactory.getInstance().newTuple(2);
- bag = BagFactory.getInstance().newDefaultBag();
- bag.add(data);
- input = TupleFactory.getInstance().newTuple(bag);
- data.set(0, 11);
- data.set(1, 51);
- distinct.accumulate(input);
-
- data = TupleFactory.getInstance().newTuple(2);
- bag = BagFactory.getInstance().newDefaultBag();
- bag.add(data);
- input = TupleFactory.getInstance().newTuple(bag);
- data.set(0, 11);
- data.set(1, 50);
- distinct.accumulate(input);
-
- result = distinct.getValue();
-
- Assert.assertEquals(2, result.size());
-
- iter = result.iterator();
- Assert.assertEquals("(12,42)", iter.next().toString());
- Assert.assertEquals("(11,51)", iter.next().toString());
- }
-
- /**
- register $JAR_PATH
-
- define CountEach datafu.pig.bags.CountEach();
-
- data = LOAD 'input' AS (data: bag {T: tuple(v1:chararray)});
-
- data2 = FOREACH data GENERATE CountEach(data) as counted;
- --describe data2;
-
- data3 = FOREACH data2 {
- ordered = ORDER counted BY count DESC;
- GENERATE ordered;
- }
- --describe data3
-
- STORE data3 INTO 'output';
-
- */
- @Multiline
- private String countEachTest;
-
- @Test
- public void countEachTest() throws Exception
- {
- PigTest test = createPigTestFromString(countEachTest);
-
- writeLinesToFile("input",
- "({(A),(B),(A),(C),(A),(B)})");
-
- test.runScript();
-
- assertOutput(test, "data3",
- "({((A),3),((B),2),((C),1)})");
- }
-
- @Test
- public void countEachExecAndAccumulateTest() throws Exception
- {
- for (int c=0; c<2; c++)
- {
- CountEach countEach = new CountEach("flatten");
-
- DataBag bag = BagFactory.getInstance().newDefaultBag();
- {
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, "A");
- bag.add(t);
- }
- {
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, "B");
- bag.add(t);
- }
- {
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, "B");
- bag.add(t);
- }
- {
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, "C");
- bag.add(t);
- }
- {
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, "A");
- bag.add(t);
- }
- {
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, "D");
- bag.add(t);
- }
-
- DataBag output = null;
-
- if (c == 0)
- {
- Tuple input = TupleFactory.getInstance().newTuple(1);
- input.set(0, bag);
-
- System.out.println("Testing exec");
- output = countEach.exec(input);
- }
- else
- {
- System.out.println("Testing accumulate");
- for (Tuple t : bag)
- {
- DataBag tb = BagFactory.getInstance().newDefaultBag();
- tb.add(t);
- Tuple input = TupleFactory.getInstance().newTuple(1);
- input.set(0, tb);
- countEach.accumulate(input);
- }
-
- output = countEach.getValue();
-
- countEach.cleanup();
- Assert.assertEquals(0, countEach.getValue().size());
- }
-
- System.out.println(output.toString());
-
- Assert.assertEquals(4, output.size());
- Set<String> found = new HashSet<String>();
- for (Tuple t : output)
- {
- String key = (String)t.get(0);
- found.add(key);
- if (key == "A")
- {
- Assert.assertEquals(2, t.get(1));
- }
- else if (key == "B")
- {
- Assert.assertEquals(2, t.get(1));
- }
- else if (key == "C")
- {
- Assert.assertEquals(1, t.get(1));
- }
- else if (key == "D")
- {
- Assert.assertEquals(1, t.get(1));
- }
- else
- {
- Assert.fail("Unexpected: " + key);
- }
- }
- Assert.assertEquals(4, found.size());
- }
- }
-
- /**
- register $JAR_PATH
-
- define CountEach datafu.pig.bags.CountEach('flatten');
-
- data = LOAD 'input' AS (data: bag {T: tuple(v1:chararray)});
-
- data2 = FOREACH data GENERATE CountEach(data) as counted;
- --describe data2;
-
- data3 = FOREACH data2 {
- ordered = ORDER counted BY count DESC;
- GENERATE ordered;
- }
- --describe data3
-
- STORE data3 INTO 'output';
-
- */
- @Multiline
- private String countEachFlattenTest;
-
- @Test
- public void countEachFlattenTest() throws Exception
- {
- PigTest test = createPigTestFromString(countEachFlattenTest);
-
- writeLinesToFile("input",
- "({(A),(B),(A),(C),(A),(B)})");
-
- test.runScript();
-
- assertOutput(test, "data3",
- "({(A,3),(B,2),(C,1)})");
- }
-
- /**
- register $JAR_PATH
-
- define BagLeftOuterJoin datafu.pig.bags.BagLeftOuterJoin();
-
- data = LOAD 'input' AS (outer_key:chararray, bag1:bag{T:tuple(k:chararray,v:chararray)}, bag2:bag{T:tuple(k:chararray,v:chararray)}, bag3:bag{T:tuple(k3:chararray,v3:chararray)});
- describe data;
-
- data2 = FOREACH data GENERATE
- outer_key,
- BagLeftOuterJoin(bag1, 'k', bag2, 'k', bag3, 'k3') as joined1,
- BagLeftOuterJoin(bag1, 'k', bag3, 'k3', bag2, 'k') as joined2; --this will break without UDF signature and pig < 0.11
- describe data2;
-
- STORE data2 INTO 'output';
-
- */
- @Multiline
- private String bagLeftOuterJoinTest;
-
- @Test
- public void bagLeftOuterJoinTest() throws Exception
- {
- PigTest test = createPigTestFromString(bagLeftOuterJoinTest);
-
- writeLinesToFile("input",
- "1\t{(K1,A1),(K2,B1),(K3,C1)}\t{(K1,A2),(K2,B2),(K2,B22)}\t{(K1,A3),(K3,C3),(K4,D3)}");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "(1,{(K1,A1,K1,A2,K1,A3),(K2,B1,K2,B2,,),(K2,B1,K2,B22,,),(K3,C1,,,K3,C3)},{(K1,A1,K1,A3,K1,A2),(K2,B1,,,K2,B2),(K2,B1,,,K2,B22),(K3,C1,K3,C3,,)})");
- }
-
- /**
- register $JAR_PATH
-
- define BagUnion datafu.pig.bags.BagConcat();
-
- data = LOAD 'input' AS (input_bag: bag {T: tuple(inner_bag: bag {T2: tuple(k: int, v: chararray)})});
- describe data;
-
- data2 = FOREACH data GENERATE BagUnion(input_bag) as unioned;
- describe data2;
-
- STORE data INTO 'output';
-
- */
- @Multiline
- private String bagUnionTest;
-
- @Test
- public void bagUnionTest() throws Exception
- {
- PigTest test = createPigTestFromString(bagUnionTest);
- writeLinesToFile("input", "({({(1,A),(1,B)}),({(2,A),(2,B),(2,C)}),({(3,A)})}");
- test.runScript();
- assertOutput(test, "data2", "({(1,A),(1,B),(2,A),(2,B),(2,C),(3,A)})");
- }
-
- /**
- register $JAR_PATH
-
- define BagGroup datafu.pig.bags.BagGroup();
-
- data = LOAD 'input' AS (input_bag: bag {T: tuple(k: int, v: chararray)});
- describe data;
-
- data2 = FOREACH data GENERATE BagGroup(input_bag, input_bag.k) as grouped;
- describe data2;
-
- data3 = FOREACH data2 {
- ordered = ORDER grouped BY group;
- GENERATE
- ordered as grouped;
- }
- describe data3;
-
- STORE data INTO 'output';
-
- */
- @Multiline
- private String bagGroupSingleTest;
-
- @Test
- public void bagGroupSingleTest() throws Exception
- {
- PigTest test = createPigTestFromString(bagGroupSingleTest);
- writeLinesToFile("input", "({(1,A),(1,B),(2,A),(2,B),(2,C),(3,A)})");
- test.runScript();
- getLinesForAlias(test, "data2", true);
- assertOutput(test, "data3", "({(1,{(1,A),(1,B)}),(2,{(2,A),(2,B),(2,C)}),(3,{(3,A)})})");
- }
-
- /**
- register $JAR_PATH
-
- define BagGroup datafu.pig.bags.BagGroup();
-
- data = LOAD 'input' AS (input_bag: bag {T: tuple(k: int, k2: chararray, v: int)});
- describe data;
-
- data2 = FOREACH data GENERATE BagGroup(input_bag, input_bag.(k, k2)) as grouped;
- describe data2;
-
- data3 = FOREACH data2 {
- ordered = ORDER grouped BY group;
- GENERATE
- ordered as grouped;
- }
- describe data3;
-
- STORE data INTO 'output';
-
- */
- @Multiline
- private String bagGroupMultipleTest;
-
- @Test
- public void bagGroupMultipleTest() throws Exception
- {
- PigTest test = createPigTestFromString(bagGroupMultipleTest);
- writeLinesToFile("input", "({(1,A,1),(1,B,1),(1,A,2),(2,A,1),(2,B,1),(2,C,1),(3,A,1)})");
- test.runScript();
- getLinesForAlias(test, "data2", true);
- assertOutput(test, "data3", "({((1,A),{(1,A,1),(1,A,2)}),((1,B),{(1,B,1)}),((2,A),{(2,A,1)}),((2,B),{(2,B,1)}),((2,C),{(2,C,1)}),((3,A),{(3,A,1)})})");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/geo/GeoTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/geo/GeoTests.java b/test/pig/datafu/test/pig/geo/GeoTests.java
deleted file mode 100644
index 39c7b61..0000000
--- a/test/pig/datafu/test/pig/geo/GeoTests.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.geo;
-
-import static org.testng.Assert.*;
-
-import java.util.List;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.test.pig.PigTests;
-
-public class GeoTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- define HaversineDistInMiles datafu.pig.geo.HaversineDistInMiles();
-
- data = LOAD 'input' AS (lat1:double,lng1:double,lat2:double,lng2:double);
-
- data2 = FOREACH data GENERATE HaversineDistInMiles(lat1,lng1,lat2,lng2);
-
- STORE data2 INTO 'output';
- */
- @Multiline
- private String haversineTest;
-
- @Test
- public void haversineTest() throws Exception
- {
- PigTest test = createPigTestFromString(haversineTest);
-
- // Approximate latitude and longitude for major cities from maps.google.com
- double[] la = {34.040143,-118.243103};
- double[] tokyo = {35.637209,139.65271};
- double[] ny = {40.716038,-73.99498};
- double[] paris = {48.857713,2.342491};
- double[] sydney = {-33.872696,151.195221};
-
- this.writeLinesToFile("input",
- coords(la,tokyo),
- coords(ny,tokyo),
- coords(ny,sydney),
- coords(ny,paris));
-
- test.runScript();
-
- List<Tuple> distances = this.getLinesForAlias(test, "data2");
-
- // ensure distance is within 20 miles of expected (distances found online)
- assertWithin(5478.0, distances.get(0), 20.0); // la <-> tokyo
- assertWithin(6760.0, distances.get(1), 20.0); // ny <-> tokyo
- assertWithin(9935.0, distances.get(2), 20.0); // ny <-> sydney
- assertWithin(3635.0, distances.get(3), 20.0); // ny <-> paris
-
- }
-
- private void assertWithin(double expected, Tuple actual, double maxDiff) throws Exception
- {
- Double actualVal = (Double)actual.get(0);
- assertTrue(Math.abs(expected-actualVal) < maxDiff);
- }
-
- private String coords(double[] coords1, double[] coords2)
- {
- assertTrue(coords1.length == 2);
- assertTrue(coords2.length == 2);
- return String.format("%f\t%f\t%f\t%f", coords1[0], coords1[1], coords2[0], coords2[1]);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/hash/HashTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/hash/HashTests.java b/test/pig/datafu/test/pig/hash/HashTests.java
deleted file mode 100644
index ba19344..0000000
--- a/test/pig/datafu/test/pig/hash/HashTests.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.hash;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.test.pig.PigTests;
-
-public class HashTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- define MD5 datafu.pig.hash.MD5();
-
- data_in = LOAD 'input' as (val:chararray);
-
- data_out = FOREACH data_in GENERATE MD5(val) as val;
-
- STORE data_out INTO 'output';
- */
- @Multiline private String md5Test;
-
- @Test
- public void md5Test() throws Exception
- {
- PigTest test = createPigTestFromString(md5Test);
-
- writeLinesToFile("input",
- "ladsljkasdglk",
- "lkadsljasgjskdjks",
- "aladlasdgjks");
-
- test.runScript();
-
- assertOutput(test, "data_out",
- "(d9a82575758bb4978949dc0659205cc6)",
- "(9ec37f02fae0d8d6a7f4453a62272f1f)",
- "(cb94139a8b9f3243e68a898ec6bd9b3d)");
- }
-
- /**
- register $JAR_PATH
-
- define SHA256 datafu.pig.hash.SHA('256');
-
- data_in = LOAD 'input' as (val:chararray);
-
- data_out = FOREACH data_in GENERATE SHA256(val) as val;
-
- STORE data_out INTO 'output';
- */
- @Multiline private String sha256Test;
-
- @Test
- public void sha256Test() throws Exception
- {
- PigTest test = createPigTestFromString(sha256Test);
-
- writeLinesToFile("input",
- "ladsljkasdglk",
- "lkadsljasgjskdjks",
- "aladlasdgjks");
-
- test.runScript();
-
- assertOutput(test, "data_out",
- "(70ebaf99c4d8ff8860869e50be2d46afbf150b883f66b50a76ee81cdc802242b)",
- "(f22e3c744a9ade0fa591d28c55392035248b391c9ee4c77ebfeaf6558c8c0dac)",
- "(49420b42e764178830783d4520aea56b759f325c1d1167f5640ded91f33f3e69)");
- }
-
- /**
- register $JAR_PATH
-
- define SHA512 datafu.pig.hash.SHA('512');
-
- data_in = LOAD 'input' as (val:chararray);
-
- data_out = FOREACH data_in GENERATE SHA512(val) as val;
-
- STORE data_out INTO 'output';
- */
- @Multiline private String sha512Test;
-
- @Test
- public void sha512Test() throws Exception
- {
- PigTest test = createPigTestFromString(sha512Test);
-
- writeLinesToFile("input",
- "ladsljkasdglk",
- "lkadsljasgjskdjks",
- "aladlasdgjks");
-
- test.runScript();
-
- assertOutput(test, "data_out",
- "(f681dbd89dfc9edf00f68107ed81b4b7c89abdf84337921785d13d9189937a43decbc264b5013d396a102b18564c39595732c43d6d4cc99473f6d6d7101ecf87)",
- "(85c130c8636c052e52a2ca091a92d0bb98ee361adcbeeebbd6af978a593b2486a22ac1e7352c683035cfa28de8eee3402adc6760ad54c5c7eda122c5124766bd)",
- "(3b82af5c08c9ab70523abf56db244eaa6740fa8c356e3a41bb5225560c0949b14b417c8d56e72cc26d5682400e0420a556e692c41ea82855013e8b7bae5fb0fb)");
- }
-
- /**
- register $JAR_PATH
-
- define MD5 datafu.pig.hash.MD5('base64');
-
- data_in = LOAD 'input' as (val:chararray);
-
- data_out = FOREACH data_in GENERATE MD5(val) as val;
-
- STORE data_out INTO 'output';
- */
- @Multiline private String md5Base64Test;
-
- @Test
- public void md5Base64Test() throws Exception
- {
- PigTest test = createPigTestFromString(md5Base64Test);
-
- writeLinesToFile("input",
- "ladsljkasdglk",
- "lkadsljasgjskdjks",
- "aladlasdgjks");
-
- test.runScript();
-
- assertOutput(test, "data_out",
- "(2agldXWLtJeJSdwGWSBcxg==)",
- "(nsN/Avrg2Nan9EU6YicvHw==)",
- "(y5QTmoufMkPmiomOxr2bPQ==)");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/linkanalysis/PageRankImplTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/linkanalysis/PageRankImplTests.java b/test/pig/datafu/test/pig/linkanalysis/PageRankImplTests.java
deleted file mode 100644
index 01d540f..0000000
--- a/test/pig/datafu/test/pig/linkanalysis/PageRankImplTests.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.linkanalysis;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.testng.annotations.Test;
-
-public class PageRankImplTests
-{
- @Test
- public void wikipediaGraphInMemoryTest() throws Exception {
- System.out.println();
- System.out.println("Starting wikipediaGraphInMemoryTest");
-
- datafu.pig.linkanalysis.PageRankImpl graph = new datafu.pig.linkanalysis.PageRankImpl();
-
- String[] edges = getWikiExampleEdges();
-
- Map<String,Integer> nodeIdsMap = loadGraphFromEdgeList(graph, edges);
-
- // Without dangling node handling we will not get the true page rank since the total rank will
- // not add to 1.0. Without dangling node handling some of the page rank drains out of the graph.
- graph.enableDanglingNodeHandling();
-
- performIterations(graph, 150, 1e-18f);
-
- String[] expectedRanks = getWikiExampleExpectedRanks();
-
- Map<String,Float> expectedRanksMap = parseExpectedRanks(expectedRanks);
-
- validateExpectedRanks(graph, nodeIdsMap, expectedRanksMap);
- }
-
- @Test
- public void wikipediaGraphDiskCacheTest() throws Exception {
- System.out.println();
- System.out.println("Starting wikipediaGraphDiskCacheTest");
-
- datafu.pig.linkanalysis.PageRankImpl graph = new datafu.pig.linkanalysis.PageRankImpl();
-
- String[] edges = getWikiExampleEdges();
-
- graph.enableEdgeDiskCaching();
- graph.setEdgeCachingThreshold(5);
-
- Map<String,Integer> nodeIdsMap = loadGraphFromEdgeList(graph, edges);
-
- assert graph.isUsingEdgeDiskCache() : "Expected disk cache to be used";
-
- // Without dangling node handling we will not get the true page rank since the total rank will
- // not add to 1.0. Without dangling node handling some of the page rank drains out of the graph.
- graph.enableDanglingNodeHandling();
-
- performIterations(graph, 150, 1e-18f);
-
- String[] expectedRanks = getWikiExampleExpectedRanks();
-
- Map<String,Float> expectedRanksMap = parseExpectedRanks(expectedRanks);
-
- validateExpectedRanks(graph, nodeIdsMap, expectedRanksMap);
- }
-
- @Test(groups="perf")
- public void hubAndSpokeInMemoryTest() throws Exception {
- System.out.println();
- System.out.println("Starting hubAndSpokeInMemoryTest");
-
- datafu.pig.linkanalysis.PageRankImpl graph = new datafu.pig.linkanalysis.PageRankImpl();
-
- String[] edges = getHubAndSpokeEdges();
-
- Map<String,Integer> nodeIdsMap = loadGraphFromEdgeList(graph, edges);
-
- graph.enableDanglingNodeHandling();
-
- performIterations(graph, 150, 1e-18f);
-
- // no need to validate, this is just a perf test for runtime comparison
- }
-
- @Test(groups="perf")
- public void hubAndSpokeDiskCacheTest() throws Exception {
- System.out.println();
- System.out.println("Starting hubAndSpokeDiskCacheTest");
-
- datafu.pig.linkanalysis.PageRankImpl graph = new datafu.pig.linkanalysis.PageRankImpl();
-
- String[] edges = getHubAndSpokeEdges();
-
- graph.enableEdgeDiskCaching();
- graph.setEdgeCachingThreshold(5);
-
- Map<String,Integer> nodeIdsMap = loadGraphFromEdgeList(graph, edges);
-
- graph.enableDanglingNodeHandling();
-
- performIterations(graph, 150, 1e-18f);
-
- // no need to validate, this is just a perf test for runtime comparison
- }
-
- private String[] getHubAndSpokeEdges()
- {
- int count = 50000;
- String[] edges = new String[count];
-
- for (int i=0; i<count; i++)
- {
- edges[i] = String.format("S%d H", i);
- }
- return edges;
- }
-
- public static String[] getWikiExampleEdges()
- {
- // graph taken from:
- // http://en.wikipedia.org/wiki/PageRank
- String[] edges = {
- "B C",
- "C B",
- "D A",
- "D B",
- "E D",
- "E B",
- "E F",
- "F E",
- "F B",
- "P1 B",
- "P1 E",
- "P2 B",
- "P2 E",
- "P3 B",
- "P3 E",
- "P4 E",
- "P5 E"
- };
- return edges;
- }
-
- public static String[] getWikiExampleExpectedRanks()
- {
- // these ranks come from the Wikipedia page:
- // http://en.wikipedia.org/wiki/PageRank
- String[] expectedRanks = {
- "A 3.3",
- "B 38.4",
- "C 34.3",
- "D 3.9",
- "E 8.1",
- "F 3.9",
- "P1 1.6",
- "P2 1.6",
- "P3 1.6",
- "P4 1.6",
- "P5 1.6"
- };
- return expectedRanks;
- }
-
- private Map<String,Integer> loadGraphFromEdgeList(datafu.pig.linkanalysis.PageRankImpl graph, String[] edges) throws IOException
- {
- Map<Integer,ArrayList<Map<String,Object>>> nodeEdgesMap = new HashMap<Integer,ArrayList<Map<String,Object>>>();
- Map<String,Integer> nodeIdsMap = new HashMap<String,Integer>();
-
- for (String edge : edges)
- {
- String[] parts = edge.split(" ");
- assert parts.length == 2 : "Expected two parts";
-
- int sourceId = getOrCreateId(parts[0], nodeIdsMap);
- int destId = getOrCreateId(parts[1], nodeIdsMap);
-
- Map<String,Object> edgeMap = new HashMap<String,Object>();
- edgeMap.put("weight", 1.0);
- edgeMap.put("dest", destId);
-
- ArrayList<Map<String,Object>> nodeEdges = null;
-
- if (nodeEdgesMap.containsKey(sourceId))
- {
- nodeEdges = nodeEdgesMap.get(sourceId);
- }
- else
- {
- nodeEdges = new ArrayList<Map<String,Object>>();
- nodeEdgesMap.put(sourceId, nodeEdges);
- }
-
- nodeEdges.add(edgeMap);
- }
-
- for (Map.Entry<Integer, ArrayList<Map<String,Object>>> e : nodeEdgesMap.entrySet())
- {
- graph.addNode(e.getKey(), e.getValue());
- }
-
- return nodeIdsMap;
- }
-
- private void performIterations(datafu.pig.linkanalysis.PageRankImpl graph, int maxIters, float tolerance) throws IOException
- {
- System.out.println(String.format("Beginning iteration (maxIters = %d, tolerance=%e)", maxIters, tolerance));
-
- System.out.println("Initializing graph");
- long startTime = System.nanoTime();
- graph.init();
- System.out.println(String.format("Done, took %f ms", (System.nanoTime() - startTime)/10.0e6));
-
- float totalDiff;
- int iter = 0;
-
- System.out.println("Beginning iterations");
- startTime = System.nanoTime();
- do
- {
- totalDiff = graph.nextIteration();
- iter++;
- } while(iter < maxIters && totalDiff > tolerance);
- System.out.println(String.format("Done, took %f ms", (System.nanoTime() - startTime)/10.0e6));
- }
-
- private void validateExpectedRanks(datafu.pig.linkanalysis.PageRankImpl graph, Map<String,Integer> nodeIds, Map<String,Float> expectedRanks)
- {
- System.out.println("Validating page rank results");
-
- for (Map.Entry<String,Integer> e : nodeIds.entrySet())
- {
- float rank = graph.getNodeRank(e.getValue());
-
- float expectedRank = expectedRanks.get(e.getKey());
- // require 0.1% accuracy
- assert (Math.abs(expectedRank - rank*100.0f) < 0.1) : String.format("Did not get expected rank for %s", e.getKey());
- }
-
- System.out.println("All ranks match expected");
- }
-
- public static Map<String,Float> parseExpectedRanks(String[] expectedRanks)
- {
- Map<String,Float> expectedRanksMap = new HashMap<String,Float>();
- for (String expectedRankString : expectedRanks)
- {
- String[] parts = expectedRankString.split(" ");
- assert parts.length == 2 : "Expected two parts";
- String name = parts[0];
- Float expectedRank = Float.parseFloat(parts[1]);
- expectedRanksMap.put(name, expectedRank);
- }
- return expectedRanksMap;
- }
-
- private Integer getOrCreateId(String name, Map<String,Integer> nodeIds)
- {
- if (nodeIds.containsKey(name))
- {
- return nodeIds.get(name);
- }
- else
- {
- Integer id = nodeIds.size();
- nodeIds.put(name, id);
- return id;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/linkanalysis/PageRankTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/linkanalysis/PageRankTests.java b/test/pig/datafu/test/pig/linkanalysis/PageRankTests.java
deleted file mode 100644
index 371d2cc..0000000
--- a/test/pig/datafu/test/pig/linkanalysis/PageRankTests.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.linkanalysis;
-
-
-import static org.testng.Assert.*;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-
-import datafu.test.pig.linkanalysis.PageRankImplTests;
-import datafu.test.pig.PigTests;
-
-public class PageRankTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- -- Need to enable dangling node handling since the Wikipedia example has them,
- -- otherwise the ranks won't be right.
- define PageRank datafu.pig.linkanalysis.PageRank('dangling_nodes','true');
-
- data = LOAD 'input' AS (topic:INT,source:INT,dest:INT,weight:DOUBLE);
-
- data_grouped = GROUP data by (topic,source);
-
- data_grouped = foreach data_grouped {
- generate group.topic as topic, group.source as source, data.(dest,weight) as edges;
- };
-
- data_grouped2 = GROUP data_grouped by topic;
- data_grouped2 = foreach data_grouped2 {
- generate group as topic, FLATTEN(PageRank(data_grouped.(source,edges))) as (source,rnk);
- };
-
- data_grouped3 = FOREACH data_grouped2 GENERATE
- topic,
- source,
- rnk;
-
- STORE data_grouped3 INTO 'output';
-
-
- */
- @Multiline private String pageRankTest;
-
- @Test
- public void pigPageRankTest() throws Exception
- {
- PigTest test = createPigTestFromString(pageRankTest);
-
- String[] edges = PageRankImplTests.getWikiExampleEdges();
-
- Map<String,Integer> nodeIds = new HashMap<String,Integer>();
- Map<Integer,String> nodeIdsReversed = new HashMap<Integer,String>();
- Map<String,Float> expectedRanks = PageRankImplTests.parseExpectedRanks(PageRankImplTests.getWikiExampleExpectedRanks());
-
- File f = new File(System.getProperty("user.dir"), "input").getAbsoluteFile();
- if (f.exists())
- {
- f.delete();
- }
-
- FileWriter writer = new FileWriter(f);
- BufferedWriter bufferedWriter = new BufferedWriter(writer);
-
- for (String edge : edges)
- {
- String[] edgeParts = edge.split(" ");
- String source = edgeParts[0];
- String dest = edgeParts[1];
- if (!nodeIds.containsKey(source))
- {
- int id = nodeIds.size();
- nodeIds.put(source,id);
- nodeIdsReversed.put(id, source);
- }
- if (!nodeIds.containsKey(dest))
- {
- int id = nodeIds.size();
- nodeIds.put(dest,id);
- nodeIdsReversed.put(id, dest);
- }
- Integer sourceId = nodeIds.get(source);
- Integer destId = nodeIds.get(dest);
-
- StringBuffer sb = new StringBuffer();
-
- sb.append("1\t"); // topic
- sb.append(sourceId.toString() + "\t");
- sb.append(destId.toString() + "\t");
- sb.append("1.0\n"); // weight
-
- bufferedWriter.write(sb.toString());
- }
-
- bufferedWriter.close();
-
- test.runScript();
- Iterator<Tuple> tuples = test.getAlias("data_grouped3");
-
- System.out.println("Final node ranks:");
- int nodeCount = 0;
- while (tuples.hasNext())
- {
- Tuple nodeTuple = tuples.next();
-
- Integer topic = (Integer)nodeTuple.get(0);
- Integer nodeId = (Integer)nodeTuple.get(1);
- Float nodeRank = (Float)nodeTuple.get(2);
-
- assertEquals(1, topic.intValue());
-
- System.out.println(String.format("%d => %f", nodeId, nodeRank));
-
- Float expectedNodeRank = expectedRanks.get(nodeIdsReversed.get(nodeId));
-
- assertTrue(Math.abs(expectedNodeRank - nodeRank * 100.0f) < 0.1,
- String.format("expected: %f, actual: %f", expectedNodeRank, nodeRank));
-
- nodeCount++;
- }
-
- assertEquals(nodeIds.size(),nodeCount);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/random/NumberTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/random/NumberTests.java b/test/pig/datafu/test/pig/random/NumberTests.java
deleted file mode 100644
index 867c27c..0000000
--- a/test/pig/datafu/test/pig/random/NumberTests.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.random;
-
-import static org.testng.Assert.*;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.test.pig.PigTests;
-
-public class NumberTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- define RandInt datafu.pig.random.RandInt();
-
- data = LOAD 'input' AS (key:INT);
- data2 = FOREACH data GENERATE key, RandInt($MIN,$MAX) as val;
-
- STORE data2 INTO 'output';
-
- */
- @Multiline private String randomIntRangeTest;
-
- /**
- * Test the RandomIntRange UDF. The main purpose is to make sure it can be used in a Pig script.
- * Also the range of output values is tested.
- *
- * @throws Exception
- */
- @Test
- public void randomIntRangeTest() throws Exception
- {
- PigTest test = createPigTestFromString(randomIntRangeTest,
- "MIN=1", "MAX=10");
-
- List<String> input = new ArrayList<String>();
- for (int i=0; i<100; i++)
- {
- input.add(String.format("(%d)", i));
- }
-
- writeLinesToFile("input",
- input.toArray(new String[0]));
-
- test.runScript();
-
- List<Tuple> tuples = getLinesForAlias(test, "data2", false);
- for (Tuple tuple : tuples)
- {
- Integer randValue = (Integer)tuple.get(1);
- assertTrue(randValue >= 1);
- assertTrue(randValue <= 10);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/random/UUIDTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/random/UUIDTests.java b/test/pig/datafu/test/pig/random/UUIDTests.java
deleted file mode 100644
index e199760..0000000
--- a/test/pig/datafu/test/pig/random/UUIDTests.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package datafu.test.pig.random;
-
-import datafu.test.pig.PigTests;
-import junit.framework.Assert;
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import java.util.*;
-
-import static org.testng.Assert.assertTrue;
-
-public class UUIDTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- define RandomUUID datafu.pig.random.RandomUUID();
-
- data = LOAD 'input' AS (key: chararray);
- DUMP data
-
- data2 = FOREACH data GENERATE key, RandomUUID() as val;
- DUMP data2
-
- STORE data2 INTO 'output';
- */
- @Multiline private String randomUUIDTest;
-
- /**
- * Test the RandomUUID UDF. The main purpose is to make sure it can be used in a Pig script.
- * Also the range of length of values is tested.
- *
- * @throws Exception
- */
- @Test
- public void randomUUIDTest() throws Exception
- {
- PigTest test = createPigTestFromString(randomUUIDTest);
-
- writeLinesToFile("input",
- "input1",
- "input2",
- "input3");
-
- List<Tuple> tuples = getLinesForAlias(test, "data2", true);
- Set<UUID> set = new HashSet<UUID>();
- for (Tuple tuple : tuples)
- {
- set.add(UUID.fromString((String)tuple.get(1)));
- }
- Assert.assertEquals(set.size(), 3);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/test/pig/datafu/test/pig/sampling/SamplingTests.java
----------------------------------------------------------------------
diff --git a/test/pig/datafu/test/pig/sampling/SamplingTests.java b/test/pig/datafu/test/pig/sampling/SamplingTests.java
deleted file mode 100644
index 9209133..0000000
--- a/test/pig/datafu/test/pig/sampling/SamplingTests.java
+++ /dev/null
@@ -1,523 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package datafu.test.pig.sampling;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import junit.framework.Assert;
-
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.pig.backend.executionengine.ExecException;
-import org.apache.pig.data.BagFactory;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
-import org.apache.pig.pigunit.PigTest;
-import org.testng.annotations.Test;
-
-import datafu.pig.sampling.ReservoirSample;
-import datafu.pig.sampling.SampleByKey;
-import datafu.pig.sampling.WeightedSample;
-import datafu.test.pig.PigTests;
-
-
-public class SamplingTests extends PigTests
-{
- /**
- register $JAR_PATH
-
- define WeightedSample datafu.pig.sampling.WeightedSample('1');
-
- data = LOAD 'input' AS (A: bag {T: tuple(v1:chararray,v2:INT)});
-
- data2 = FOREACH data GENERATE WeightedSample(A,1);
- --describe data2;
-
- STORE data2 INTO 'output';
-
- */
- @Multiline
- private String weightedSampleTest;
-
- @Test
- public void weightedSampleTest() throws Exception
- {
- PigTest test = createPigTestFromString(weightedSampleTest);
-
- writeLinesToFile("input",
- "({(a, 100),(b, 1),(c, 5),(d, 2)})");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "({(a,100),(c,5),(b,1),(d,2)})");
- }
-
- /**
- register $JAR_PATH
-
- define WeightedSample datafu.pig.sampling.WeightedSample('1');
-
- data = LOAD 'input' AS (A: bag {T: tuple(v1:chararray,v2:INT)});
-
- data2 = FOREACH data GENERATE WeightedSample(A,1,3);
- --describe data2;
-
- STORE data2 INTO 'output';
- */
- @Multiline
- private String weightedSampleLimitTest;
-
- @Test
- public void weightedSampleLimitTest() throws Exception
- {
- PigTest test = createPigTestFromString(weightedSampleLimitTest);
-
- writeLinesToFile("input",
- "({(a, 100),(b, 1),(c, 5),(d, 2)})");
-
- test.runScript();
-
- assertOutput(test, "data2",
- "({(a,100),(c,5),(b,1)})");
- }
-
- @Test
- public void weightedSampleLimitExecTest() throws IOException
- {
- WeightedSample sampler = new WeightedSample();
-
- DataBag bag = BagFactory.getInstance().newDefaultBag();
- for (int i=0; i<100; i++)
- {
- Tuple t = TupleFactory.getInstance().newTuple(2);
- t.set(0, i);
- t.set(1, 1); // score is equal for all
- bag.add(t);
- }
-
- Tuple input = TupleFactory.getInstance().newTuple(3);
- input.set(0, bag);
- input.set(1, 1); // use index 1 for score
- input.set(2, 10); // get 10 items
-
- DataBag result = sampler.exec(input);
-
- Assert.assertEquals(10, result.size());
-
- // all must be found, no repeats
- Set<Integer> found = new HashSet<Integer>();
- for (Tuple t : result)
- {
- Integer i = (Integer)t.get(0);
- System.out.println(i);
- Assert.assertTrue(i>=0 && i<100);
- Assert.assertFalse(String.format("Found duplicate of %d",i), found.contains(i));
- found.add(i);
- }
- }
-
- /**
- register $JAR_PATH
-
- DEFINE SampleByKey datafu.pig.sampling.SampleByKey('0.5', 'salt2.5');
-
- data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
- sampled = FILTER data BY SampleByKey(A_id);
-
- STORE sampled INTO 'output';
-
- */
- @Multiline
- private String sampleByKeyTest;
-
- @Test
- public void sampleByKeyTest() throws Exception
- {
- PigTest test = createPigTestFromString(sampleByKeyTest);
-
- writeLinesToFile("input",
- "A1\tB1\t1","A1\tB1\t4","A1\tB3\t4","A1\tB4\t4",
- "A2\tB1\t4","A2\tB2\t4",
- "A3\tB1\t3","A3\tB1\t1","A3\tB3\t77",
- "A4\tB1\t3","A4\tB2\t3","A4\tB3\t59","A4\tB4\t29",
- "A5\tB1\t4",
- "A6\tB2\t3","A6\tB2\t55","A6\tB3\t1",
- "A7\tB1\t39","A7\tB2\t27","A7\tB3\t85",
- "A8\tB1\t4","A8\tB2\t45",
- "A9\tB3\t92", "A9\tB1\t42","A9\tB2\t1","A9\tB3\t0",
- "A10\tB1\t7","A10\tB2\t23","A10\tB3\t1","A10\tB4\t41","A10\tB5\t52");
-
- test.runScript();
- assertOutput(test, "sampled",
- "(A4,B1,3)","(A4,B2,3)","(A4,B3,59)","(A4,B4,29)",
- "(A5,B1,4)",
- "(A6,B2,3)","(A6,B2,55)","(A6,B3,1)",
- "(A7,B1,39)","(A7,B2,27)","(A7,B3,85)",
- "(A10,B1,7)","(A10,B2,23)","(A10,B3,1)","(A10,B4,41)","(A10,B5,52)");
- }
-
- /**
- register $JAR_PATH
-
- DEFINE SampleByKey datafu.pig.sampling.SampleByKey('0.5', 'salt2.5');
-
- data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
- sampled = FILTER data BY SampleByKey(A_id, B_id);
-
- STORE sampled INTO 'output';
-
- */
- @Multiline
- private String sampleByKeyMultipleKeyTest;
-
- @Test
- public void sampleByKeyMultipleKeyTest() throws Exception
- {
- PigTest test = createPigTestFromString(sampleByKeyMultipleKeyTest);
-
- writeLinesToFile("input",
- "A1\tB1\t1","A1\tB1\t4",
- "A1\tB3\t4",
- "A1\tB4\t4",
- "A2\tB1\t4",
- "A2\tB2\t4",
- "A3\tB1\t3","A3\tB1\t1",
- "A3\tB3\t77",
- "A4\tB1\t3",
- "A4\tB2\t3",
- "A4\tB3\t59",
- "A4\tB4\t29",
- "A5\tB1\t4",
- "A6\tB2\t3","A6\tB2\t55",
- "A6\tB3\t1",
- "A7\tB1\t39",
- "A7\tB2\t27",
- "A7\tB3\t85",
- "A8\tB1\t4",
- "A8\tB2\t45",
- "A9\tB3\t92","A9\tB3\t0",
- "A9\tB6\t42","A9\tB5\t1",
- "A10\tB1\t7",
- "A10\tB2\t23","A10\tB2\t1","A10\tB2\t31",
- "A10\tB6\t41",
- "A10\tB7\t52");
- test.runScript();
- assertOutput(test, "sampled",
- "(A1,B1,1)","(A1,B1,4)",
- "(A1,B4,4)",
- "(A2,B1,4)",
- "(A2,B2,4)",
- "(A3,B1,3)","(A3,B1,1)",
- "(A4,B4,29)",
- "(A5,B1,4)",
- "(A6,B3,1)",
- "(A7,B1,39)",
- "(A8,B1,4)",
- "(A9,B3,92)","(A9,B3,0)",
- "(A10,B2,23)","(A10,B2,1)","(A10,B2,31)"
- );
-
- }
-
- @Test
- public void sampleByKeyExecTest() throws Exception
- {
- SampleByKey sampler = new SampleByKey("0.10", "thesalt");
-
- Map<Integer,Integer> valuesPerKey = new HashMap<Integer,Integer>();
-
- // 10,000 keys total
- for (int i=0; i<10000; i++)
- {
- // 5 values per key
- for (int j=0; j<5; j++)
- {
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, i);
- if (sampler.exec(t))
- {
- if (valuesPerKey.containsKey(i))
- {
- valuesPerKey.put(i, valuesPerKey.get(i)+1);
- }
- else
- {
- valuesPerKey.put(i, 1);
- }
- }
- }
- }
-
- // 10% sample, so should have roughly 1000 keys
- Assert.assertTrue(Math.abs(1000-valuesPerKey.size()) < 50);
-
- // every value should be present for the same key
- for (Map.Entry<Integer, Integer> pair : valuesPerKey.entrySet())
- {
- Assert.assertEquals(5, pair.getValue().intValue());
- }
- }
-
- /**
- register $JAR_PATH
-
- DEFINE ReservoirSample datafu.pig.sampling.ReservoirSample('$RESERVOIR_SIZE');
-
- data = LOAD 'input' AS (A_id:chararray, B_id:chararray, C:int);
- sampled = FOREACH (GROUP data ALL) GENERATE ReservoirSample(data) as sample_data;
- sampled = FOREACH sampled GENERATE COUNT(sample_data) AS sample_count;
- STORE sampled INTO 'output';
-
- */
- @Multiline
- private String reservoirSampleTest;
-
- @Test
- public void reservoirSampleTest() throws Exception
- {
-
- writeLinesToFile("input",
- "A1\tB1\t1",
- "A1\tB1\t4",
- "A1\tB3\t4",
- "A1\tB4\t4",
- "A2\tB1\t4",
- "A2\tB2\t4",
- "A3\tB1\t3",
- "A3\tB1\t1",
- "A3\tB3\t77",
- "A4\tB1\t3",
- "A4\tB2\t3",
- "A4\tB3\t59",
- "A4\tB4\t29",
- "A5\tB1\t4",
- "A6\tB2\t3",
- "A6\tB2\t55",
- "A6\tB3\t1",
- "A7\tB1\t39",
- "A7\tB2\t27",
- "A7\tB3\t85",
- "A8\tB1\t4",
- "A8\tB2\t45",
- "A9\tB3\t92",
- "A9\tB3\t0",
- "A9\tB6\t42",
- "A9\tB5\t1",
- "A10\tB1\t7",
- "A10\tB2\t23",
- "A10\tB2\t1",
- "A10\tB2\t31",
- "A10\tB6\t41",
- "A10\tB7\t52");
-
- for(int i=10; i<=30; i=i+10){
- int reservoirSize = i ;
- PigTest test = createPigTestFromString(reservoirSampleTest, "RESERVOIR_SIZE="+reservoirSize);
- test.runScript();
- assertOutput(test, "sampled", "("+reservoirSize+")");
- }
- }
-
- /**
- register $JAR_PATH
-
- DEFINE ReservoirSample datafu.pig.sampling.ReservoirSample('$RESERVOIR_SIZE');
- DEFINE Assert datafu.pig.util.Assert();
-
- data = LOAD 'input' AS (A_id:int, B_id:chararray, C:int);
- sampled = FOREACH (GROUP data BY A_id) GENERATE group as A_id, ReservoirSample(data.(B_id,C)) as sample_data;
- sampled = FILTER sampled BY Assert((SIZE(sample_data) <= $RESERVOIR_SIZE ? 1 : 0), 'must be <= $RESERVOIR_SIZE');
- sampled = FOREACH sampled GENERATE A_id, FLATTEN(sample_data);
- STORE sampled INTO 'output';
-
- */
- @Multiline
- private String reservoirSampleGroupTest;
-
- /**
- * Verifies that ReservoirSample works when data grouped by a key.
- * In particular it ensures that the reservoir is not reused across keys.
- *
- * <p>
- * This confirms the fix for DATAFU-11.
- * </p>
- *
- * @throws Exception
- */
- @Test
- public void reservoirSampleGroupTest() throws Exception
- {
- // first value is the key. last value matches the key so we can
- // verify the register is reset for each key. values should not
- // bleed across to other keys.
- writeLinesToFile("input",
- "1\tB1\t1",
- "1\tB1\t1",
- "1\tB3\t1",
- "1\tB4\t1",
- "2\tB1\t2",
- "2\tB2\t2",
- "3\tB1\t3",
- "3\tB1\t3",
- "3\tB3\t3",
- "4\tB1\t4",
- "4\tB2\t4",
- "4\tB3\t4",
- "4\tB4\t4",
- "5\tB1\t5",
- "6\tB2\t6",
- "6\tB2\t6",
- "6\tB3\t6",
- "7\tB1\t7",
- "7\tB2\t7",
- "7\tB3\t7",
- "8\tB1\t8",
- "8\tB2\t8",
- "9\tB3\t9",
- "9\tB3\t9",
- "9\tB6\t9",
- "9\tB5\t9",
- "10\tB1\t10",
- "10\tB2\t10",
- "10\tB2\t10",
- "10\tB2\t10",
- "10\tB6\t10",
- "10\tB7\t10");
-
- for(int i=1; i<=3; i++) {
- int reservoirSize = i ;
- PigTest test = createPigTestFromString(reservoirSampleGroupTest, "RESERVOIR_SIZE="+reservoirSize);
- test.runScript();
-
- List<Tuple> tuples = getLinesForAlias(test, "sampled");
-
- for (Tuple tuple : tuples)
- {
- Assert.assertEquals(((Number)tuple.get(0)).intValue(), ((Number)tuple.get(2)).intValue());
- }
- }
- }
-
- @Test
- public void reservoirSampleExecTest() throws IOException
- {
- ReservoirSample sampler = new ReservoirSample("10");
-
- DataBag bag = BagFactory.getInstance().newDefaultBag();
- for (int i=0; i<100; i++)
- {
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, i);
- bag.add(t);
- }
-
- Tuple input = TupleFactory.getInstance().newTuple(bag);
-
- DataBag result = sampler.exec(input);
-
- Assert.assertEquals(10, result.size());
-
- // all must be found, no repeats
- Set<Integer> found = new HashSet<Integer>();
- for (Tuple t : result)
- {
- Integer i = (Integer)t.get(0);
- System.out.println(i);
- Assert.assertTrue(i>=0 && i<100);
- Assert.assertFalse(String.format("Found duplicate of %d",i), found.contains(i));
- found.add(i);
- }
- }
-
- @Test
- public void reservoirSampleAccumulateTest() throws IOException
- {
- ReservoirSample sampler = new ReservoirSample("10");
-
- for (int i=0; i<100; i++)
- {
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, i);
- DataBag bag = BagFactory.getInstance().newDefaultBag();
- bag.add(t);
- Tuple input = TupleFactory.getInstance().newTuple(bag);
- sampler.accumulate(input);
- }
-
- DataBag result = sampler.getValue();
-
- Assert.assertEquals(10, result.size());
-
- // all must be found, no repeats
- Set<Integer> found = new HashSet<Integer>();
- for (Tuple t : result)
- {
- Integer i = (Integer)t.get(0);
- System.out.println(i);
- Assert.assertTrue(i>=0 && i<100);
- Assert.assertFalse(String.format("Found duplicate of %d",i), found.contains(i));
- found.add(i);
- }
- }
-
- @Test
- public void reservoirSampleAlgebraicTest() throws IOException
- {
- ReservoirSample.Initial initialSampler = new ReservoirSample.Initial("10");
- ReservoirSample.Intermediate intermediateSampler = new ReservoirSample.Intermediate("10");
- ReservoirSample.Final finalSampler = new ReservoirSample.Final("10");
-
- DataBag bag = BagFactory.getInstance().newDefaultBag();
- for (int i=0; i<100; i++)
- {
- Tuple t = TupleFactory.getInstance().newTuple(1);
- t.set(0, i);
- bag.add(t);
- }
-
- Tuple input = TupleFactory.getInstance().newTuple(bag);
-
- Tuple intermediateTuple = initialSampler.exec(input);
- DataBag intermediateBag = BagFactory.getInstance().newDefaultBag(Arrays.asList(intermediateTuple));
- intermediateTuple = intermediateSampler.exec(TupleFactory.getInstance().newTuple(intermediateBag));
- intermediateBag = BagFactory.getInstance().newDefaultBag(Arrays.asList(intermediateTuple));
- DataBag result = finalSampler.exec(TupleFactory.getInstance().newTuple(intermediateBag));
-
- Assert.assertEquals(10, result.size());
-
- // all must be found, no repeats
- Set<Integer> found = new HashSet<Integer>();
- for (Tuple t : result)
- {
- Integer i = (Integer)t.get(0);
- System.out.println(i);
- Assert.assertTrue(i>=0 && i<100);
- Assert.assertFalse(String.format("Found duplicate of %d",i), found.contains(i));
- found.add(i);
- }
- }
-}
[16/19] DATAFU-27 Migrate build system to Gradle
Posted by wv...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/sets/SetIntersect.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/sets/SetIntersect.java b/datafu-pig/src/main/java/datafu/pig/sets/SetIntersect.java
new file mode 100644
index 0000000..0ba586e
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/sets/SetIntersect.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.sets;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.PriorityQueue;
+
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+
+/**
+ * Computes the set intersection of two or more bags. Duplicates are eliminated. <b>The input bags must be sorted.</b>
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * define SetIntersect datafu.pig.sets.SetIntersect();
+ *
+ * -- input:
+ * -- ({(1,10),(2,20),(3,30),(4,40)},{(2,20),(4,40),(8,80)})
+ * input = LOAD 'input' AS (B1:bag{T:tuple(val1:int,val2:int)},B2:bag{T:tuple(val1:int,val2:int)});
+ *
+ * input = FOREACH input {
+ * B1 = ORDER B1 BY val1 ASC, val2 ASC;
+ * B2 = ORDER B2 BY val1 ASC, val2 ASC;
+ *
+ * -- output:
+ * -- ({(2,20),(4,40)})
+ * GENERATE SetIntersect(B1,B2);
+ * }
+ * }</pre>
+ */
+public class SetIntersect extends SetOperationsBase
+{
+ private static final BagFactory bagFactory = BagFactory.getInstance();
+
+ static class pair implements Comparable<pair>
+ {
+ final Iterator<Tuple> it;
+ Tuple data;
+
+ pair(Iterator<Tuple> it)
+ {
+ this.it = it;
+ this.data = it.next();
+ }
+
+ @Override
+ public int compareTo(pair o)
+ {
+ return this.data.compareTo(o.data);
+ }
+ }
+
+ private PriorityQueue<pair> load_bags(Tuple input) throws IOException
+ {
+ PriorityQueue<pair> pq = new PriorityQueue<pair>(input.size());
+
+ for (int i=0; i < input.size(); i++) {
+ Object o = input.get(i);
+ if (!(o instanceof DataBag))
+ throw new RuntimeException("parameters must be databags");
+ Iterator<Tuple> inputIterator= ((DataBag) o).iterator();
+ if(inputIterator.hasNext())
+ pq.add(new pair(inputIterator));
+ }
+ return pq;
+ }
+
+ public boolean all_equal(PriorityQueue<pair> pq)
+ {
+ Object o = pq.peek().data;
+ for (pair p : pq) {
+ if (!o.equals(p.data))
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public DataBag exec(Tuple input) throws IOException
+ {
+ DataBag outputBag = bagFactory.newDefaultBag();
+ PriorityQueue<pair> pq = load_bags(input);
+ if(pq.size() != input.size())
+ return outputBag; // one or more input bags were empty
+ Tuple last_data = null;
+
+ while (true) {
+ if (pq.peek().data.compareTo(last_data) != 0 && all_equal(pq)) {
+ last_data = pq.peek().data;
+ outputBag.add(last_data);
+ }
+
+ pair p = pq.poll();
+ if (!p.it.hasNext())
+ break;
+ Tuple nextData = p.it.next();
+ // algorithm assumes data is in order
+ if (p.data.compareTo(nextData) > 0)
+ {
+ throw new RuntimeException("Out of order!");
+ }
+ p.data = nextData;
+ pq.offer(p);
+ }
+
+ return outputBag;
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/sets/SetOperationsBase.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/sets/SetOperationsBase.java b/datafu-pig/src/main/java/datafu/pig/sets/SetOperationsBase.java
new file mode 100644
index 0000000..c9997f8
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/sets/SetOperationsBase.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.sets;
+
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * Base class for set operations.
+ *
+ * @author "Matthew Hayes <mh...@linkedin.com>"
+ *
+ */
+public abstract class SetOperationsBase extends EvalFunc<DataBag>
+{
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try {
+ for (Schema.FieldSchema fieldSchema : input.getFields())
+ {
+ if (fieldSchema.type != DataType.BAG)
+ {
+ throw new RuntimeException("Expected a bag but got: " + DataType.findTypeName(fieldSchema.type));
+ }
+ }
+
+ Schema bagSchema = input.getField(0).schema;
+
+ Schema outputSchema = new Schema(new Schema.FieldSchema(getSchemaName(this.getClass()
+ .getName()
+ .toLowerCase(), input),
+ bagSchema,
+ DataType.BAG));
+
+ return outputSchema;
+ }
+ catch (Exception e) {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/sets/SetUnion.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/sets/SetUnion.java b/datafu-pig/src/main/java/datafu/pig/sets/SetUnion.java
new file mode 100644
index 0000000..49fd99c
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/sets/SetUnion.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.sets;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+
+/**
+ * Computes the set union of two or more bags. Duplicates are eliminated.
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ * define SetUnion datafu.pig.sets.SetUnion();
+ *
+ * -- input:
+ * -- ({(2,20),(3,30),(4,40)},{(1,10),(2,20),(4,40),(8,80)})
+ * input = LOAD 'input' AS (B1:bag{T:tuple(val1:int,val2:int)},B2:bag{T:tuple(val1:int,val2:int)});
+ *
+ * -- output:
+ * -- ({(2,20),(3,30),(4,40),(1,10),(8,80)})
+ * output = FOREACH input GENERATE SetUnion(B1,B2);
+ * }
+ * </pre>
+ */
+public class SetUnion extends SetOperationsBase
+{
+ private static final BagFactory bagFactory = BagFactory.getInstance();
+ private static final TupleFactory tupleFactory = TupleFactory.getInstance();
+
+ @Override
+ public DataBag exec(Tuple input) throws IOException
+ {
+ DataBag outputBag = bagFactory.newDistinctBag();
+
+ try {
+ for (int i=0; i < input.size(); i++) {
+ Object o = input.get(i);
+ if (!(o instanceof DataBag))
+ throw new RuntimeException("parameters must be databags");
+
+ DataBag inputBag = (DataBag) o;
+ for (Tuple elem : inputBag) {
+ outputBag.add(elem);
+ }
+ }
+
+ return outputBag;
+ }
+ catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/sets/package-info.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/sets/package-info.java b/datafu-pig/src/main/java/datafu/pig/sets/package-info.java
new file mode 100644
index 0000000..2c5b087
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/sets/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * UDFs for set operations such as intersect and union.
+ */
+package datafu.pig.sets;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/DoubleVAR.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/DoubleVAR.java b/datafu-pig/src/main/java/datafu/pig/stats/DoubleVAR.java
new file mode 100644
index 0000000..ab8eed2
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/DoubleVAR.java
@@ -0,0 +1,373 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.pig.Accumulator;
+import org.apache.pig.Algebraic;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.FuncSpec;
+import org.apache.pig.PigException;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.backend.executionengine.ExecException;
+
+
+/**
+* Use {@link VAR}
+*/
+public class DoubleVAR extends EvalFunc<Double> implements Algebraic, Accumulator<Double> {
+ private static TupleFactory mTupleFactory = TupleFactory.getInstance();
+
+ @Override
+ public Double exec(Tuple input) throws IOException {
+ try {
+ Double sum = sum(input);
+ Double sumSquare = sumSquare(input);
+
+ if(sum == null) {
+ // either we were handed an empty bag or a bag
+ // filled with nulls - return null in this case
+ return null;
+ }
+ long count = count(input);
+
+ Double var = null;
+ if (count > 0){
+ Double avg = new Double(sum / count);
+ Double avgSquare = new Double(sumSquare / count);
+ var = avgSquare - avg*avg;
+ }
+
+ return var;
+ } catch (ExecException ee) {
+ throw ee;
+ }
+ }
+
+ public String getInitial() {
+ return Initial.class.getName();
+ }
+
+ public String getIntermed() {
+ return Intermediate.class.getName();
+ }
+
+ public String getFinal() {
+ return Final.class.getName();
+ }
+
+ static public class Initial extends EvalFunc<Tuple> {
+ @Override
+ public Tuple exec(Tuple input) throws IOException {
+ Tuple t = mTupleFactory.newTuple(3);
+ try {
+ // input is a bag with one tuple containing
+ // the column we are trying to get variance
+ DataBag bg = (DataBag) input.get(0);
+ Double d = null;
+ Iterator<Tuple> iter = bg.iterator();
+ if(iter.hasNext()) {
+ Tuple tp = iter.next();
+ d = (Double)tp.get(0);
+ }
+
+ if (iter.hasNext())
+ {
+ throw new RuntimeException("Expected only one tuple in bag");
+ }
+
+ if (d == null){
+ t.set(0, null);
+ t.set(1, null);
+ t.set(2, 0L);
+ }
+ else {
+ t.set(0, d);
+ t.set(1, d*d);
+ t.set(2, 1L);
+ }
+ return t;
+ } catch(NumberFormatException nfe) {
+ nfe.printStackTrace();
+ // invalid input,
+ // treat this input as null
+ try {
+ t.set(0, null);
+ t.set(1, null);
+ t.set(2, 0L);
+ } catch (ExecException e) {
+ throw e;
+ }
+ return t;
+ } catch (ExecException ee) {
+ ee.printStackTrace();
+ throw ee;
+ } catch (Exception e) {
+ e.printStackTrace();
+ int errCode = 2106;
+ String msg = "Error while computing variance in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+ }
+
+ }
+ }
+
+ static public class Intermediate extends EvalFunc<Tuple> {
+ @Override
+ public Tuple exec(Tuple input) throws IOException {
+ try {
+ DataBag b = (DataBag)input.get(0);
+ return combine(b);
+ } catch (ExecException ee) {
+ throw ee;
+ } catch (Exception e) {
+ int errCode = 2106;
+ String msg = "Error while computing variacne in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+
+ }
+ }
+ }
+
+ static public class Final extends EvalFunc<Double> {
+ @Override
+ public Double exec(Tuple input) throws IOException {
+ try {
+ DataBag b = (DataBag)input.get(0);
+ Tuple combined = combine(b);
+
+ Double sum = (Double)combined.get(0);
+ Double sumSquare = (Double)combined.get(1);
+ if(sum == null) {
+ return null;
+ }
+ Long count = (Long)combined.get(2);
+
+ Double var = null;
+
+ if (count > 0) {
+ Double avg = new Double(sum / count);
+ Double avgSquare = new Double(sumSquare / count);
+ var = avgSquare - avg*avg;
+ }
+ return var;
+ } catch (ExecException ee) {
+ throw ee;
+ } catch (Exception e) {
+ int errCode = 2106;
+ String msg = "Error while computing variance in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+ }
+ }
+ }
+
+ static protected Tuple combine(DataBag values) throws ExecException{
+ double sum = 0;
+ double sumSquare = 0;
+ long totalCount = 0;
+
+ // combine is called from Intermediate and Final
+ // In either case, Initial would have been called
+ // before and would have sent in valid tuples
+ // Hence we don't need to check if incoming bag
+ // is empty
+
+ Tuple output = mTupleFactory.newTuple(3);
+ boolean sawNonNull = false;
+ for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
+ Tuple t = it.next();
+ Double d = (Double)t.get(0);
+ Double dSquare = (Double)t.get(1);
+ Long count = (Long)t.get(2);
+
+ // we count nulls in var as contributing 0
+ // a departure from SQL for performance of
+ // COUNT() which implemented by just inspecting
+ // size of the bag
+ if(d == null) {
+ d = 0.0;
+ dSquare = 0.0;
+ } else {
+ sawNonNull = true;
+ }
+ sum += d;
+ sumSquare += dSquare;
+ totalCount += count;
+ }
+ if(sawNonNull) {
+ output.set(0, new Double(sum));
+ output.set(1, new Double(sumSquare));
+ } else {
+ output.set(0, null);
+ output.set(1, null);
+ }
+ output.set(2, Long.valueOf(totalCount));
+ return output;
+ }
+
+ static protected long count(Tuple input) throws ExecException {
+ DataBag values = (DataBag)input.get(0);
+ long cnt = 0;
+ Iterator<Tuple> it = values.iterator();
+ while (it.hasNext()){
+ Tuple t = (Tuple)it.next();
+ if (t != null && t.size() > 0 && t.get(0) != null)
+ cnt ++;
+ }
+
+ return cnt;
+ }
+
+ static protected Double sum(Tuple input) throws ExecException, IOException {
+ DataBag values = (DataBag)input.get(0);
+
+ // if we were handed an empty bag, return NULL
+ if(values.size() == 0) {
+ return null;
+ }
+
+ double sum = 0;
+ boolean sawNonNull = false;
+ for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
+ Tuple t = it.next();
+ try{
+ Double d= (Double)t.get(0);
+ if (d == null) continue;
+ sawNonNull = true;
+ sum += d;
+ }catch(RuntimeException exp) {
+ int errCode = 2103;
+ String msg = "Problem while computing sum of values.";
+ throw new ExecException(msg, errCode, PigException.BUG, exp);
+ }
+ }
+
+ if(sawNonNull) {
+ return new Double(sum);
+ } else {
+ return null;
+ }
+ }
+
+ static protected Double sumSquare(Tuple input) throws ExecException, IOException {
+ DataBag values = (DataBag)input.get(0);
+
+ // if we were handed an empty bag, return NULL
+ if(values.size() == 0) {
+ return null;
+ }
+
+ double sumSquare = 0;
+ boolean sawNonNull = false;
+ for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
+ Tuple t = it.next();
+ try{
+ Double d = (Double)t.get(0);
+ if (d == null) continue;
+ sawNonNull = true;
+ sumSquare += d*d;
+ }catch(RuntimeException exp) {
+ int errCode = 2103;
+ String msg = "Problem while computing sum of squared values.";
+ throw new ExecException(msg, errCode, PigException.BUG, exp);
+ }
+ }
+
+ if(sawNonNull) {
+ return new Double(sumSquare);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public Schema outputSchema(Schema input) {
+ return new Schema(new Schema.FieldSchema(null, DataType.DOUBLE));
+ }
+
+ /* Accumulator interface implementation */
+ private Double intermediateSumSquare = null;
+ private Double intermediateSum = null;
+ private Long intermediateCount = null;
+
+ @Override
+ public void accumulate(Tuple b) throws IOException {
+ try {
+ Double sum = sum(b);
+ if(sum == null) {
+ return;
+ }
+
+ Double sumSquare = sumSquare(b);
+ if(sumSquare == null) {
+ return;
+ }
+
+ // set default values
+ if (intermediateSum == null || intermediateCount == null) {
+ intermediateSumSquare = 0.0;
+ intermediateSum = 0.0;
+ intermediateCount = (long) 0;
+ }
+
+ long count = (Long)count(b);
+
+ if (count > 0) {
+ intermediateCount += count;
+ intermediateSum += sum;
+ intermediateSumSquare += sumSquare;
+ }
+ } catch (ExecException ee) {
+ throw ee;
+ } catch (Exception e) {
+ int errCode = 2106;
+ String msg = "Error while computing variance in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+ }
+ }
+
+ @Override
+ public void cleanup() {
+ intermediateSumSquare = null;
+ intermediateSum = null;
+ intermediateCount = null;
+ }
+
+ @Override
+ public Double getValue() {
+ Double var = null;
+ if (intermediateCount != null && intermediateCount > 0) {
+ Double avg = new Double(intermediateSum / intermediateCount);
+ Double avgSquare = new Double(intermediateSumSquare / intermediateCount);
+ var = avgSquare - avg*avg;
+ }
+ return var;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/FloatVAR.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/FloatVAR.java b/datafu-pig/src/main/java/datafu/pig/stats/FloatVAR.java
new file mode 100644
index 0000000..8a697e0
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/FloatVAR.java
@@ -0,0 +1,375 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.pig.Accumulator;
+import org.apache.pig.Algebraic;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.FuncSpec;
+import org.apache.pig.PigException;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.backend.executionengine.ExecException;
+
+
+/**
+* Use {@link VAR}
+*/
+public class FloatVAR extends EvalFunc<Double> implements Algebraic, Accumulator<Double> {
+ private static TupleFactory mTupleFactory = TupleFactory.getInstance();
+
+ @Override
+ public Double exec(Tuple input) throws IOException {
+ try {
+ Double sum = sum(input);
+ Double sumSquare = sumSquare(input);
+
+ if(sum == null) {
+ // either we were handed an empty bag or a bag
+ // filled with nulls - return null in this case
+ return null;
+ }
+ long count = count(input);
+
+ Double var = null;
+ if (count > 0){
+ Double avg = new Double(sum / count);
+ Double avgSquare = new Double(sumSquare / count);
+ var = avgSquare - avg*avg;
+ }
+
+ return var;
+ } catch (ExecException ee) {
+ throw ee;
+ }
+ }
+
+ public String getInitial() {
+ return Initial.class.getName();
+ }
+
+ public String getIntermed() {
+ return Intermediate.class.getName();
+ }
+
+ public String getFinal() {
+ return Final.class.getName();
+ }
+
+ static public class Initial extends EvalFunc<Tuple> {
+ @Override
+ public Tuple exec(Tuple input) throws IOException {
+ Tuple t = mTupleFactory.newTuple(3);
+ try {
+ // input is a bag with one tuple containing
+ // the column we are trying to get variance
+ DataBag bg = (DataBag) input.get(0);
+ Float f = null;
+ Iterator<Tuple> iter = bg.iterator();
+ if(iter.hasNext()) {
+ Tuple tp = iter.next();
+ f = (Float)tp.get(0);
+ }
+
+ if (iter.hasNext())
+ {
+ throw new RuntimeException("Expected only one tuple in bag");
+ }
+
+ Double d = f!= null ? new Double(f): null;
+
+ if (f == null){
+ t.set(0, null);
+ t.set(1, null);
+ t.set(2, 0L);
+ }
+ else {
+ t.set(0, d);
+ t.set(1, d*d);
+ t.set(2, 1L);
+ }
+ return t;
+ } catch(NumberFormatException nfe) {
+ nfe.printStackTrace();
+ // invalid input,
+ // treat this input as null
+ try {
+ t.set(0, null);
+ t.set(1, null);
+ t.set(2, 0L);
+ } catch (ExecException e) {
+ throw e;
+ }
+ return t;
+ } catch (ExecException ee) {
+ ee.printStackTrace();
+ throw ee;
+ } catch (Exception e) {
+ e.printStackTrace();
+ int errCode = 2106;
+ String msg = "Error while computing variance in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+ }
+
+ }
+ }
+
+ static public class Intermediate extends EvalFunc<Tuple> {
+ @Override
+ public Tuple exec(Tuple input) throws IOException {
+ try {
+ DataBag b = (DataBag)input.get(0);
+ return combine(b);
+ } catch (ExecException ee) {
+ throw ee;
+ } catch (Exception e) {
+ int errCode = 2106;
+ String msg = "Error while computing variacne in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+
+ }
+ }
+ }
+
+ static public class Final extends EvalFunc<Double> {
+ @Override
+ public Double exec(Tuple input) throws IOException {
+ try {
+ DataBag b = (DataBag)input.get(0);
+ Tuple combined = combine(b);
+
+ Double sum = (Double)combined.get(0);
+ Double sumSquare = (Double)combined.get(1);
+ if(sum == null) {
+ return null;
+ }
+ Long count = (Long)combined.get(2);
+
+ Double var = null;
+
+ if (count > 0) {
+ Double avg = new Double(sum / count);
+ Double avgSquare = new Double(sumSquare / count);
+ var = avgSquare - avg*avg;
+ }
+ return var;
+ } catch (ExecException ee) {
+ throw ee;
+ } catch (Exception e) {
+ int errCode = 2106;
+ String msg = "Error while computing variance in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+ }
+ }
+ }
+
+ static protected Tuple combine(DataBag values) throws ExecException{
+ double sum = 0;
+ double sumSquare = 0;
+ long totalCount = 0;
+
+ // combine is called from Intermediate and Final
+ // In either case, Initial would have been called
+ // before and would have sent in valid tuples
+ // Hence we don't need to check if incoming bag
+ // is empty
+
+ Tuple output = mTupleFactory.newTuple(3);
+ boolean sawNonNull = false;
+ for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
+ Tuple t = it.next();
+ Double d = (Double)t.get(0);
+ Double dSquare = (Double)t.get(1);
+ Long count = (Long)t.get(2);
+
+ // we count nulls in var as contributing 0
+ // a departure from SQL for performance of
+ // COUNT() which implemented by just inspecting
+ // size of the bag
+ if(d == null) {
+ d = 0.0;
+ dSquare = 0.0;
+ } else {
+ sawNonNull = true;
+ }
+ sum += d;
+ sumSquare += dSquare;
+ totalCount += count;
+ }
+ if(sawNonNull) {
+ output.set(0, new Double(sum));
+ output.set(1, new Double(sumSquare));
+ } else {
+ output.set(0, null);
+ output.set(1, null);
+ }
+ output.set(2, Long.valueOf(totalCount));
+ return output;
+ }
+
+ static protected long count(Tuple input) throws ExecException {
+ DataBag values = (DataBag)input.get(0);
+ long cnt = 0;
+ Iterator<Tuple> it = values.iterator();
+ while (it.hasNext()){
+ Tuple t = (Tuple)it.next();
+ if (t != null && t.size() > 0 && t.get(0) != null)
+ cnt ++;
+ }
+
+ return cnt;
+ }
+
+ static protected Double sum(Tuple input) throws ExecException, IOException {
+ DataBag values = (DataBag)input.get(0);
+
+ // if we were handed an empty bag, return NULL
+ if(values.size() == 0) {
+ return null;
+ }
+
+ double sum = 0;
+ boolean sawNonNull = false;
+ for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
+ Tuple t = it.next();
+ try{
+ Float f= (Float)t.get(0);
+ if (f == null) continue;
+ sawNonNull = true;
+ sum += f;
+ }catch(RuntimeException exp) {
+ int errCode = 2103;
+ String msg = "Problem while computing sum of values.";
+ throw new ExecException(msg, errCode, PigException.BUG, exp);
+ }
+ }
+
+ if(sawNonNull) {
+ return new Double(sum);
+ } else {
+ return null;
+ }
+ }
+
+ static protected Double sumSquare(Tuple input) throws ExecException, IOException {
+ DataBag values = (DataBag)input.get(0);
+
+ // if we were handed an empty bag, return NULL
+ if(values.size() == 0) {
+ return null;
+ }
+
+ double sumSquare = 0;
+ boolean sawNonNull = false;
+ for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
+ Tuple t = it.next();
+ try{
+ Float f = (Float)t.get(0);
+ if (f == null) continue;
+ sawNonNull = true;
+ sumSquare += f*f;
+ }catch(RuntimeException exp) {
+ int errCode = 2103;
+ String msg = "Problem while computing sum of squared values.";
+ throw new ExecException(msg, errCode, PigException.BUG, exp);
+ }
+ }
+
+ if(sawNonNull) {
+ return new Double(sumSquare);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public Schema outputSchema(Schema input) {
+ return new Schema(new Schema.FieldSchema(null, DataType.DOUBLE));
+ }
+
+ /* Accumulator interface implementation */
+ private Double intermediateSumSquare = null;
+ private Double intermediateSum = null;
+ private Long intermediateCount = null;
+
+ @Override
+ public void accumulate(Tuple b) throws IOException {
+ try {
+ Double sum = sum(b);
+ if(sum == null) {
+ return;
+ }
+
+ Double sumSquare = sumSquare(b);
+ if(sumSquare == null) {
+ return;
+ }
+
+ // set default values
+ if (intermediateSum == null || intermediateCount == null) {
+ intermediateSumSquare = 0.0;
+ intermediateSum = 0.0;
+ intermediateCount = (long) 0;
+ }
+
+ long count = (Long)count(b);
+
+ if (count > 0) {
+ intermediateCount += count;
+ intermediateSum += sum;
+ intermediateSumSquare += sumSquare;
+ }
+ } catch (ExecException ee) {
+ throw ee;
+ } catch (Exception e) {
+ int errCode = 2106;
+ String msg = "Error while computing variance in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+ }
+ }
+
+ @Override
+ public void cleanup() {
+ intermediateSumSquare = null;
+ intermediateSum = null;
+ intermediateCount = null;
+ }
+
+ @Override
+ public Double getValue() {
+ Double var = null;
+ if (intermediateCount != null && intermediateCount > 0) {
+ Double avg = new Double(intermediateSum / intermediateCount);
+ Double avgSquare = new Double(intermediateSumSquare / intermediateCount);
+ var = avgSquare - avg*avg;
+ }
+ return var;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/HyperLogLogPlusPlus.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/HyperLogLogPlusPlus.java b/datafu-pig/src/main/java/datafu/pig/stats/HyperLogLogPlusPlus.java
new file mode 100644
index 0000000..0ebd94f
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/HyperLogLogPlusPlus.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats;
+
+import java.io.IOException;
+
+import org.apache.pig.AccumulatorEvalFunc;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+/**
+ * A UDF that applies the HyperLogLog++ cardinality estimation algorithm.
+ *
+ * <p>
+ * This uses the implementation of HyperLogLog++ from <a href="https://github.com/addthis/stream-lib" target="_blank">stream-lib</a>.
+ * The HyperLogLog++ algorithm is an enhanced version of HyperLogLog as described in
+ * <a href="http://static.googleusercontent.com/external_content/untrusted_dlcp/research.google.com/en/us/pubs/archive/40671.pdf">here</a>.
+ * </p>
+ *
+ * <p>
+ * This is a streaming implementation, and therefore the input data does not need to be sorted.
+ * </p>
+ *
+ * @author mhayes
+ *
+ */
+public class HyperLogLogPlusPlus extends AccumulatorEvalFunc<Long>
+{
+ private com.clearspring.analytics.stream.cardinality.HyperLogLogPlus estimator;
+
+ private final int p;
+
+ /**
+ * Constructs a HyperLogLog++ estimator.
+ */
+ public HyperLogLogPlusPlus()
+ {
+ this("20");
+ }
+
+ /**
+ * Constructs a HyperLogLog++ estimator.
+ *
+ * @param p precision value
+ */
+ public HyperLogLogPlusPlus(String p)
+ {
+ this.p = Integer.parseInt(p);
+ cleanup();
+ }
+
+ @Override
+ public void accumulate(Tuple arg0) throws IOException
+ {
+ DataBag inputBag = (DataBag)arg0.get(0);
+ for (Tuple t : inputBag)
+ {
+ estimator.offer(t);
+ }
+ }
+
+ @Override
+ public void cleanup()
+ {
+ this.estimator = new com.clearspring.analytics.stream.cardinality.HyperLogLogPlus(p);
+ }
+
+ @Override
+ public Long getValue()
+ {
+ return this.estimator.cardinality();
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try {
+ if (input.size() != 1)
+ {
+ throw new RuntimeException("Expected input to have only a single field");
+ }
+
+ Schema.FieldSchema inputFieldSchema = input.getField(0);
+
+ if (inputFieldSchema.type != DataType.BAG)
+ {
+ throw new RuntimeException("Expected a BAG as input");
+ }
+
+ return new Schema(new Schema.FieldSchema(null, DataType.LONG));
+ }
+ catch (FrontendException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/IntVAR.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/IntVAR.java b/datafu-pig/src/main/java/datafu/pig/stats/IntVAR.java
new file mode 100644
index 0000000..0f34d8e
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/IntVAR.java
@@ -0,0 +1,372 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.pig.Accumulator;
+import org.apache.pig.Algebraic;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.FuncSpec;
+import org.apache.pig.PigException;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.backend.executionengine.ExecException;
+
+
+/**
+* Use {@link VAR}
+*/
+public class IntVAR extends EvalFunc<Double> implements Algebraic, Accumulator<Double> {
+ private static TupleFactory mTupleFactory = TupleFactory.getInstance();
+
+ @Override
+ public Double exec(Tuple input) throws IOException {
+ try {
+ Long sum = sum(input);
+ Long sumSquare = sumSquare(input);
+
+ if(sum == null) {
+ // either we were handed an empty bag or a bag
+ // filled with nulls - return null in this case
+ return null;
+ }
+ long count = count(input);
+
+ Double var = null;
+ if (count > 0){
+ Double avg = new Double((double)sum / count);
+ Double avgSquare = new Double((double)sumSquare / count);
+ var = avgSquare - avg*avg;
+ }
+
+ return var;
+ } catch (ExecException ee) {
+ throw ee;
+ }
+ }
+
+ public String getInitial() {
+ return Initial.class.getName();
+ }
+
+ public String getIntermed() {
+ return Intermediate.class.getName();
+ }
+
+ public String getFinal() {
+ return Final.class.getName();
+ }
+
+ static public class Initial extends EvalFunc<Tuple> {
+ @Override
+ public Tuple exec(Tuple input) throws IOException {
+ Tuple t = mTupleFactory.newTuple(3);
+ try {
+ // input is a bag with one tuple containing
+ // the column we are trying to get variance
+ DataBag bg = (DataBag) input.get(0);
+ Integer i = null;
+ Iterator<Tuple> iter = bg.iterator();
+ if(iter.hasNext()) {
+ Tuple tp = iter.next();
+ i = (Integer)tp.get(0);
+ }
+
+ if (iter.hasNext())
+ {
+ throw new RuntimeException("Expected only one tuple in bag");
+ }
+
+ if (i == null) {
+ t.set(2, 0L);
+ t.set(0, null);
+ t.set(1, null);
+ }
+ else {
+ t.set(2, 1L);
+ t.set(0, (long)i);
+ t.set(1, (long)i*i);
+ }
+ return t;
+ } catch(NumberFormatException nfe) {
+ nfe.printStackTrace();
+ // invalid input,
+ // treat this input as null
+ try {
+ t.set(0, null);
+ t.set(1, null);
+ t.set(2, 0L);
+ } catch (ExecException e) {
+ throw e;
+ }
+ return t;
+ } catch (ExecException ee) {
+ ee.printStackTrace();
+ throw ee;
+ } catch (Exception e) {
+ e.printStackTrace();
+ int errCode = 2106;
+ String msg = "Error while computing variance in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+ }
+
+ }
+ }
+
+ static public class Intermediate extends EvalFunc<Tuple> {
+ @Override
+ public Tuple exec(Tuple input) throws IOException {
+ try {
+ DataBag b = (DataBag)input.get(0);
+ return combine(b);
+ } catch (ExecException ee) {
+ throw ee;
+ } catch (Exception e) {
+ int errCode = 2106;
+ String msg = "Error while computing variacne in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+ }
+ }
+ }
+
+ static public class Final extends EvalFunc<Double> {
+ @Override
+ public Double exec(Tuple input) throws IOException {
+ try {
+ DataBag b = (DataBag)input.get(0);
+ Tuple combined = combine(b);
+
+ Long sum = (Long)combined.get(0);
+ Long sumSquare = (Long)combined.get(1);
+ if(sum == null) {
+ return null;
+ }
+ Long count = (Long)combined.get(2);
+
+ Double var = null;
+
+ if (count > 0) {
+ Double avg = new Double((double)sum / count);
+ Double avgSquare = new Double((double)sumSquare / count);
+ var = avgSquare - avg*avg;
+ }
+ return var;
+ } catch (ExecException ee) {
+ throw ee;
+ } catch (Exception e) {
+ int errCode = 2106;
+ String msg = "Error while computing variance in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+ }
+ }
+ }
+
+ static protected Tuple combine(DataBag values) throws ExecException{
+ long sum = 0;
+ long sumSquare = 0;
+ long totalCount = 0;
+
+ // combine is called from Intermediate and Final
+ // In either case, Initial would have been called
+ // before and would have sent in valid tuples
+ // Hence we don't need to check if incoming bag
+ // is empty
+
+ Tuple output = mTupleFactory.newTuple(3);
+ boolean sawNonNull = false;
+ for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
+ Tuple t = it.next();
+ Long i = (Long)t.get(0);
+ Long iSquare = (Long)t.get(1);
+ Long count = (Long)t.get(2);
+
+ // we count nulls in var as contributing 0
+ // a departure from SQL for performance of
+ // COUNT() which implemented by just inspecting
+ // size of the bag
+ if(i == null) {
+ i = (long)0;
+ iSquare = (long)0;
+ } else {
+ sawNonNull = true;
+ }
+ sum += i;
+ sumSquare += iSquare;
+ totalCount += count;
+ }
+ if(sawNonNull) {
+ output.set(0, new Long(sum));
+ output.set(1, new Long(sumSquare));
+ } else {
+ output.set(0, null);
+ output.set(1, null);
+ }
+ output.set(2, Long.valueOf(totalCount));
+ return output;
+ }
+
+ static protected long count(Tuple input) throws ExecException {
+ DataBag values = (DataBag)input.get(0);
+ long cnt = 0;
+ Iterator<Tuple> it = values.iterator();
+ while (it.hasNext()){
+ Tuple t = (Tuple)it.next();
+ if (t != null && t.size() > 0 && t.get(0) != null)
+ cnt ++;
+ }
+
+ return cnt;
+ }
+
+ static protected Long sum(Tuple input) throws ExecException, IOException {
+ DataBag values = (DataBag)input.get(0);
+
+ // if we were handed an empty bag, return NULL
+ if(values.size() == 0) {
+ return null;
+ }
+
+ long sum = 0;
+ boolean sawNonNull = false;
+ for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
+ Tuple t = it.next();
+ try{
+ Integer i = (Integer)t.get(0);
+ if (i == null) continue;
+ sawNonNull = true;
+ sum += i;
+ }catch(RuntimeException exp) {
+ int errCode = 2103;
+ String msg = "Problem while computing sum of values.";
+ throw new ExecException(msg, errCode, PigException.BUG, exp);
+ }
+ }
+
+ if(sawNonNull) {
+ return new Long(sum);
+ } else {
+ return null;
+ }
+ }
+
+ static protected Long sumSquare(Tuple input) throws ExecException, IOException {
+ DataBag values = (DataBag)input.get(0);
+
+ // if we were handed an empty bag, return NULL
+ if(values.size() == 0) {
+ return null;
+ }
+
+ long sumSquare = 0;
+ boolean sawNonNull = false;
+ for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
+ Tuple t = it.next();
+ try{
+ Integer i = (Integer)t.get(0);
+ if (i == null) continue;
+ sawNonNull = true;
+ sumSquare += i*i;
+ }catch(RuntimeException exp) {
+ int errCode = 2103;
+ String msg = "Problem while computing sum of squared values.";
+ throw new ExecException(msg, errCode, PigException.BUG, exp);
+ }
+ }
+
+ if(sawNonNull) {
+ return new Long(sumSquare);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public Schema outputSchema(Schema input) {
+ return new Schema(new Schema.FieldSchema(null, DataType.DOUBLE));
+ }
+
+ /* Accumulator interface implementation */
+ private Long intermediateSumSquare = null;
+ private Long intermediateSum = null;
+ private Long intermediateCount = null;
+
+ @Override
+ public void accumulate(Tuple b) throws IOException {
+ try {
+ Long sum = sum(b);
+ if(sum == null) {
+ return;
+ }
+
+ Long sumSquare = sumSquare(b);
+ if(sumSquare == null) {
+ return;
+ }
+
+ // set default values
+ if (intermediateSum == null || intermediateCount == null) {
+ intermediateSumSquare = (long) 0;
+ intermediateSum = (long) 0;
+ intermediateCount = (long) 0;
+ }
+
+ long count = (Long)count(b);
+
+ if (count > 0) {
+ intermediateCount += count;
+ intermediateSum += sum;
+ intermediateSumSquare += sumSquare;
+ }
+ } catch (ExecException ee) {
+ throw ee;
+ } catch (Exception e) {
+ int errCode = 2106;
+ String msg = "Error while computing variance in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+ }
+ }
+
+ @Override
+ public void cleanup() {
+ intermediateSumSquare = null;
+ intermediateSum = null;
+ intermediateCount = null;
+ }
+
+ @Override
+ public Double getValue() {
+ Double var = null;
+ if (intermediateCount != null && intermediateCount > 0) {
+ Double avg = new Double((double)intermediateSum / intermediateCount);
+ Double avgSquare = new Double((double)intermediateSumSquare / intermediateCount);
+ var = avgSquare - avg*avg;
+ }
+ return var;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/LongVAR.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/LongVAR.java b/datafu-pig/src/main/java/datafu/pig/stats/LongVAR.java
new file mode 100644
index 0000000..561f15a
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/LongVAR.java
@@ -0,0 +1,373 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.pig.Accumulator;
+import org.apache.pig.Algebraic;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.FuncSpec;
+import org.apache.pig.PigException;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.backend.executionengine.ExecException;
+
+
+/**
+* Use {@link VAR}
+*/
+public class LongVAR extends EvalFunc<Double> implements Algebraic, Accumulator<Double> {
+ private static TupleFactory mTupleFactory = TupleFactory.getInstance();
+
+ @Override
+ public Double exec(Tuple input) throws IOException {
+ try {
+ Long sum = sum(input);
+ Long sumSquare = sumSquare(input);
+
+ if(sum == null) {
+ // either we were handed an empty bag or a bag
+ // filled with nulls - return null in this case
+ return null;
+ }
+ long count = count(input);
+
+ Double var = null;
+ if (count > 0){
+ Double avg = new Double((double)sum / count);
+ Double avgSquare = new Double((double)sumSquare / count);
+ var = avgSquare - avg*avg;
+ }
+
+ return var;
+ } catch (ExecException ee) {
+ throw ee;
+ }
+ }
+
+ public String getInitial() {
+ return Initial.class.getName();
+ }
+
+ public String getIntermed() {
+ return Intermediate.class.getName();
+ }
+
+ public String getFinal() {
+ return Final.class.getName();
+ }
+
+ static public class Initial extends EvalFunc<Tuple> {
+ @Override
+ public Tuple exec(Tuple input) throws IOException {
+ Tuple t = mTupleFactory.newTuple(3);
+ try {
+ // input is a bag with one tuple containing
+ // the column we are trying to get variance
+ DataBag bg = (DataBag) input.get(0);
+ Long l = null;
+ Iterator<Tuple> iter = bg.iterator();
+ if(iter.hasNext()) {
+ Tuple tp = iter.next();
+ l = (Long)tp.get(0);
+ }
+
+ if (iter.hasNext())
+ {
+ throw new RuntimeException("Expected only one tuple in bag");
+ }
+
+ if (l == null) {
+ t.set(0, null);
+ t.set(1, null);
+ t.set(2, 0L);
+ }
+ else {
+ t.set(0, l);
+ t.set(1, l*l);
+ t.set(2, 1L);
+ }
+ return t;
+ } catch(NumberFormatException nfe) {
+ nfe.printStackTrace();
+ // invalid input,
+ // treat this input as null
+ try {
+ t.set(0, null);
+ t.set(1, null);
+ t.set(2, 0L);
+ } catch (ExecException e) {
+ throw e;
+ }
+ return t;
+ } catch (ExecException ee) {
+ ee.printStackTrace();
+ throw ee;
+ } catch (Exception e) {
+ e.printStackTrace();
+ int errCode = 2106;
+ String msg = "Error while computing variance in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+ }
+
+ }
+ }
+
+ static public class Intermediate extends EvalFunc<Tuple> {
+ @Override
+ public Tuple exec(Tuple input) throws IOException {
+ try {
+ DataBag b = (DataBag)input.get(0);
+ return combine(b);
+ } catch (ExecException ee) {
+ throw ee;
+ } catch (Exception e) {
+ int errCode = 2106;
+ String msg = "Error while computing variacne in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+
+ }
+ }
+ }
+
+ static public class Final extends EvalFunc<Double> {
+ @Override
+ public Double exec(Tuple input) throws IOException {
+ try {
+ DataBag b = (DataBag)input.get(0);
+ Tuple combined = combine(b);
+
+ Long sum = (Long)combined.get(0);
+ Long sumSquare = (Long)combined.get(1);
+ if(sum == null) {
+ return null;
+ }
+ Long count = (Long)combined.get(2);
+
+ Double var = null;
+
+ if (count > 0) {
+ Double avg = new Double((double)sum / count);
+ Double avgSquare = new Double((double)sumSquare / count);
+ var = avgSquare - avg*avg;
+ }
+ return var;
+ } catch (ExecException ee) {
+ throw ee;
+ } catch (Exception e) {
+ int errCode = 2106;
+ String msg = "Error while computing variance in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+ }
+ }
+ }
+
+ static protected Tuple combine(DataBag values) throws ExecException{
+ long sum = 0;
+ long sumSquare = 0;
+ long totalCount = 0;
+
+ // combine is called from Intermediate and Final
+ // In either case, Initial would have been called
+ // before and would have sent in valid tuples
+ // Hence we don't need to check if incoming bag
+ // is empty
+
+ Tuple output = mTupleFactory.newTuple(3);
+ boolean sawNonNull = false;
+ for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
+ Tuple t = it.next();
+ Long l = (Long)t.get(0);
+ Long lSquare = (Long)t.get(1);
+ Long count = (Long)t.get(2);
+
+ // we count nulls in var as contributing 0
+ // a departure from SQL for performance of
+ // COUNT() which implemented by just inspecting
+ // size of the bag
+ if(l == null) {
+ l = (long)0;
+ lSquare = (long)0;
+ } else {
+ sawNonNull = true;
+ }
+ sum += l;
+ sumSquare += lSquare;
+ totalCount += count;
+ }
+ if(sawNonNull) {
+ output.set(0, new Long(sum));
+ output.set(1, new Long(sumSquare));
+ } else {
+ output.set(0, null);
+ output.set(1, null);
+ }
+ output.set(2, Long.valueOf(totalCount));
+ return output;
+ }
+
+ static protected long count(Tuple input) throws ExecException {
+ DataBag values = (DataBag)input.get(0);
+ long cnt = 0;
+ Iterator<Tuple> it = values.iterator();
+ while (it.hasNext()){
+ Tuple t = (Tuple)it.next();
+ if (t != null && t.size() > 0 && t.get(0) != null)
+ cnt ++;
+ }
+
+ return cnt;
+ }
+
+ static protected Long sum(Tuple input) throws ExecException, IOException {
+ DataBag values = (DataBag)input.get(0);
+
+ // if we were handed an empty bag, return NULL
+ if(values.size() == 0) {
+ return null;
+ }
+
+ long sum = 0;
+ boolean sawNonNull = false;
+ for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
+ Tuple t = it.next();
+ try{
+ Long l = (Long)t.get(0);
+ if (l == null) continue;
+ sawNonNull = true;
+ sum += l;
+ }catch(RuntimeException exp) {
+ int errCode = 2103;
+ String msg = "Problem while computing sum of values.";
+ throw new ExecException(msg, errCode, PigException.BUG, exp);
+ }
+ }
+
+ if(sawNonNull) {
+ return new Long(sum);
+ } else {
+ return null;
+ }
+ }
+
+ static protected Long sumSquare(Tuple input) throws ExecException, IOException {
+ DataBag values = (DataBag)input.get(0);
+
+ // if we were handed an empty bag, return NULL
+ if(values.size() == 0) {
+ return null;
+ }
+
+ long sumSquare = 0;
+ boolean sawNonNull = false;
+ for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
+ Tuple t = it.next();
+ try{
+ Long l = (Long)t.get(0);
+ if (l == null) continue;
+ sawNonNull = true;
+ sumSquare += l*l;
+ }catch(RuntimeException exp) {
+ int errCode = 2103;
+ String msg = "Problem while computing sum of squared values.";
+ throw new ExecException(msg, errCode, PigException.BUG, exp);
+ }
+ }
+
+ if(sawNonNull) {
+ return new Long(sumSquare);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public Schema outputSchema(Schema input) {
+ return new Schema(new Schema.FieldSchema(null, DataType.DOUBLE));
+ }
+
+ /* Accumulator interface implementation */
+ private Long intermediateSumSquare = null;
+ private Long intermediateSum = null;
+ private Long intermediateCount = null;
+
+ @Override
+ public void accumulate(Tuple b) throws IOException {
+ try {
+ Long sum = sum(b);
+ if(sum == null) {
+ return;
+ }
+
+ Long sumSquare = sumSquare(b);
+ if(sumSquare == null) {
+ return;
+ }
+
+ // set default values
+ if (intermediateSum == null || intermediateCount == null) {
+ intermediateSumSquare = (long) 0;
+ intermediateSum = (long) 0;
+ intermediateCount = (long) 0;
+ }
+
+ long count = (Long)count(b);
+
+ if (count > 0) {
+ intermediateCount += count;
+ intermediateSum += sum;
+ intermediateSumSquare += sumSquare;
+ }
+ } catch (ExecException ee) {
+ throw ee;
+ } catch (Exception e) {
+ int errCode = 2106;
+ String msg = "Error while computing variance in " + this.getClass().getSimpleName();
+ throw new ExecException(msg, errCode, PigException.BUG, e);
+ }
+ }
+
+ @Override
+ public void cleanup() {
+ intermediateSumSquare = null;
+ intermediateSum = null;
+ intermediateCount = null;
+ }
+
+ @Override
+ public Double getValue() {
+ Double var = null;
+ if (intermediateCount != null && intermediateCount > 0) {
+ Double avg = new Double((double)intermediateSum / intermediateCount);
+ Double avgSquare = new Double((double)intermediateSumSquare / intermediateCount);
+ var = avgSquare - avg*avg;
+ }
+ return var;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/MarkovPairs.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/MarkovPairs.java b/datafu-pig/src/main/java/datafu/pig/stats/MarkovPairs.java
new file mode 100644
index 0000000..d3d7632
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/MarkovPairs.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import org.apache.pig.EvalFunc;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
+
+
+/**
+ * Accepts a bag of tuples, with user supplied ordering, and generates pairs that can be used for
+ * a Markov chain analysis. For example, if you had {(1), (4), (7)}, using the default lookahead of 1, you
+ * get the pairs {
+ * ((1),(4)),
+ * ((4),(7))}
+ * A lookahead factor tells the UDF how many steps in to the future to include. so, for a,b,c with a lookahead
+ * of 2, a would be paired with both b and c.
+ * The results are ordered are returned as ordered by the caller.
+*/
+
+public class MarkovPairs extends EvalFunc<DataBag>
+{
+ private static final BagFactory bagFactory = BagFactory.getInstance();
+ private static final TupleFactory tupleFactory = TupleFactory.getInstance();
+
+ private static long lookahead_steps;
+
+ private final int SPILL_THRESHOLD = 1000000;
+
+ public MarkovPairs()
+ {
+ MarkovPairs.lookahead_steps = 1;
+ }
+
+ public MarkovPairs(String lookahead_steps)
+ {
+ MarkovPairs.lookahead_steps = Integer.valueOf(lookahead_steps);
+ }
+
+ /* start and end are inclusive. This forms transition pairs */
+ private void generatePairs(ArrayList<Tuple> input, int start, int end, DataBag outputBag)
+ throws ExecException
+ {
+ int count = 0;
+ for (int i = start; (i + 1)<= end; i++) {
+ Tuple elem1 = input.get(i);
+
+ lookahead:
+ for (int j = i+1; j <= i + lookahead_steps; j++)
+ {
+ if (j > end) break lookahead;
+ Tuple elem2 = input.get(j);
+ if (count >= SPILL_THRESHOLD) {
+ outputBag.spill();
+ count = 0;
+ }
+ outputBag.add(tupleFactory.newTuple(Arrays.asList(elem1, elem2)));
+ count ++;
+ }
+ }
+ }
+
+ @Override
+ public DataBag exec(Tuple input)
+ throws IOException
+ {
+ //things come in a tuple, in our case we have a bag (ordered views) passed. This is embedded in a length one tuple
+
+ DataBag inputBag = (DataBag) input.get(0);
+ ArrayList<Tuple> inputData = new ArrayList<Tuple>();
+
+ for (Tuple tuple : inputBag) {
+ inputData.add(tuple);
+ }
+
+ int inputSize = inputData.size();
+
+ try {
+ DataBag outputBag = bagFactory.newDefaultBag();
+
+ int startPos = 0;
+
+ int stopPos = inputSize - 1;
+ generatePairs(inputData, startPos, stopPos, outputBag);
+
+ // set startPos for the next bucket
+ startPos = stopPos + 1;
+ return outputBag;
+ }
+ catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ try {
+ Schema tupleSchema = new Schema();
+
+ FieldSchema fieldSchema = input.getField(0);
+
+ if (fieldSchema.type != DataType.BAG)
+ {
+ throw new RuntimeException(String.format("Expected input schema to be BAG, but instead found %s",
+ DataType.findTypeName(fieldSchema.type)));
+ }
+
+ FieldSchema fieldSchema2 = fieldSchema.schema.getField(0);
+
+ tupleSchema.add(new Schema.FieldSchema("elem1", fieldSchema2.schema));
+ tupleSchema.add(new Schema.FieldSchema("elem2", fieldSchema2.schema));
+ return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
+ tupleSchema,
+ DataType.BAG));
+ }
+ catch (Exception e) {
+ return null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/Median.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/Median.java b/datafu-pig/src/main/java/datafu/pig/stats/Median.java
new file mode 100644
index 0000000..e33a84e
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/Median.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats;
+
+/**
+ * Computes the {@link <a href="http://en.wikipedia.org/wiki/Median" target="_blank">median</a>}
+ * for a <b>sorted</b> input bag, using type R-2 estimation. This is a convenience wrapper around Quantile.
+ *
+ * <p>
+ * N.B., all the data is pushed to a single reducer per key, so make sure some partitioning is
+ * done (e.g., group by 'day') if the data is too large. That is, this isn't distributed median.
+ * </p>
+ *
+ * @see Quantile
+ */
+public class Median extends Quantile
+{
+ public Median()
+ {
+ super("0.5");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/Quantile.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/Quantile.java b/datafu-pig/src/main/java/datafu/pig/stats/Quantile.java
new file mode 100644
index 0000000..6fd42d3
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/Quantile.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
+
+import datafu.pig.util.SimpleEvalFunc;
+
+/**
+ * Computes {@link <a href="http://en.wikipedia.org/wiki/Quantile" target="_blank">quantiles</a>}
+ * for a <b>sorted</b> input bag, using type R-2 estimation.
+ *
+ * <p>
+ * N.B., all the data is pushed to a single reducer per key, so make sure some partitioning is
+ * done (e.g., group by 'day') if the data is too large. That is, this isn't distributed quantiles.
+ * </p>
+ *
+ * <p>
+ * Note that unlike datafu's StreamingQuantile algorithm, this implementation gives
+ * <b>exact</b> quantiles. But, it requires that the input bag to be sorted. Quantile must spill to
+ * disk when the input data is too large to fit in memory, which will contribute to longer runtimes.
+ * Because StreamingQuantile implements accumulate it can be much more efficient than Quantile for
+ * large input bags which do not fit well in memory.
+ * </p>
+ *
+ * <p>The constructor takes a single integer argument that specifies the number of evenly-spaced
+ * quantiles to compute, e.g.,</p>
+ *
+ * <ul>
+ * <li>Quantile('3') yields the min, the median, and the max
+ * <li>Quantile('5') yields the min, the 25th, 50th, 75th percentiles, and the max
+ * <li>Quantile('101') yields the min, the max, and all 99 percentiles.
+ * </ul>
+ *
+ * <p>Alternatively the constructor can take the explicit list of quantiles to compute, e.g.</p>
+ *
+ * <ul>
+ * <li>Quantile('0.0','0.5','1.0') yields the min, the median, and the max
+ * <li>Quantile('0.0','0.25','0.5','0.75','1.0') yields the min, the 25th, 50th, 75th percentiles, and the max
+ * </ul>
+ *
+ * <p>The list of quantiles need not span the entire range from 0.0 to 1.0, nor do they need to be evenly spaced, e.g.</p>
+ *
+ * <ul>
+ * <li>Quantile('0.5','0.90','0.95','0.99') yields the median, the 90th, 95th, and the 99th percentiles
+ * <li>Quantile('0.0013','0.0228','0.1587','0.5','0.8413','0.9772','0.9987') yields the 0.13th, 2.28th, 15.87th, 50th, 84.13th, 97.72nd, and 99.87th percentiles
+ * </ul>
+ *
+ * <p>
+ * Example:
+ * <pre>
+ * {@code
+ *
+ * define Quantile datafu.pig.stats.Quantile('0.0','0.5','1.0');
+
+ * -- input: 9,10,2,3,5,8,1,4,6,7
+ * input = LOAD 'input' AS (val:int);
+ *
+ * grouped = GROUP input ALL;
+ *
+ * -- produces: (1,5.5,10)
+ * quantiles = FOREACH grouped {
+ * sorted = ORDER input BY val;
+ * GENERATE Quantile(sorted);
+ * }
+ * }</pre></p>
+ *
+ * @see Median
+ * @see StreamingQuantile
+ */
+public class Quantile extends SimpleEvalFunc<Tuple>
+{
+ List<Double> quantiles;
+
+ private static class Pair<T1,T2>
+ {
+ public T1 first;
+ public T2 second;
+
+ public Pair(T1 first, T2 second) {
+ this.first = first;
+ this.second = second;
+ }
+ }
+
+ // For the output schema, label the quantiles 0, 1, 2, ... n
+ // Otherwise label the quantiles based on the quantile value.
+ // e.g. 50% quantile 0.5 will be labeled as 0_5
+ private boolean ordinalOutputSchema;
+
+ public Quantile(String... k)
+ {
+ this.quantiles = QuantileUtil.getQuantilesFromParams(k);
+
+ if (k.length == 1 && Double.parseDouble(k[0]) > 1.0)
+ {
+ this.ordinalOutputSchema = true;
+ this.quantiles = QuantileUtil.getQuantilesFromParams(k);
+ }
+ else
+ {
+ this.quantiles = QuantileUtil.getQuantilesFromParams(k);
+ }
+ }
+
+ private static Pair<Long, Long> getIndexes(double k, long N)
+ {
+ double h = N*k + 0.5;
+ long i1 = Math.min(Math.max(1, (long)Math.ceil(h - 0.5)), N);
+ long i2 = Math.min(Math.max(1, (long)Math.floor(h + 0.5)), N);
+
+ return new Pair<Long, Long>(i1, i2);
+ }
+
+ public Tuple call(DataBag bag) throws IOException
+ {
+ if (bag == null || bag.size() == 0)
+ return null;
+
+ Map<Long, Double> d = new HashMap<Long, Double>();
+ long N = bag.size(), max_id = 1;
+
+ for (double k : this.quantiles) {
+ Pair<Long, Long> idx = getIndexes(k, N);
+
+ d.put(idx.first, null);
+ d.put(idx.second, null);
+ max_id = Math.max(max_id, idx.second);
+ }
+
+ long i = 1;
+ for (Tuple t : bag) {
+ if (i > max_id)
+ break;
+
+ if (d.containsKey(i)) {
+ Object o = t.get(0);
+ if (!(o instanceof Number))
+ throw new IllegalStateException("bag must have numerical values (and be non-null)");
+ d.put(i, ((Number) o).doubleValue());
+ }
+ i++;
+ }
+
+ Tuple t = TupleFactory.getInstance().newTuple(this.quantiles.size());
+ int j = 0;
+ for (double k : this.quantiles) {
+ Pair<Long, Long> p = getIndexes(k, N);
+ double quantile = (d.get(p.first) + d.get(p.second)) / 2;
+ t.set(j, quantile);
+ j++;
+ }
+ return t;
+ }
+
+ @Override
+ public Schema outputSchema(Schema input)
+ {
+ Schema tupleSchema = new Schema();
+ if (ordinalOutputSchema)
+ {
+ for (int i = 0; i < this.quantiles.size(); i++)
+ {
+ tupleSchema.add(new Schema.FieldSchema("quantile_" + i, DataType.DOUBLE));
+ }
+ }
+ else
+ {
+ for (Double x : this.quantiles)
+ tupleSchema.add(new Schema.FieldSchema("quantile_" + x.toString().replace(".", "_"), DataType.DOUBLE));
+ }
+
+ try {
+ return new Schema(new FieldSchema(null, tupleSchema, DataType.TUPLE));
+ } catch(FrontendException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/QuantileUtil.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/QuantileUtil.java b/datafu-pig/src/main/java/datafu/pig/stats/QuantileUtil.java
new file mode 100644
index 0000000..c6fd36a
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/QuantileUtil.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats;
+
+import java.util.ArrayList;
+
+/**
+ * Methods used by {@link Quantile}.
+ *
+ * @author "Matthew Hayes <mh...@linkedin.com>"
+ *
+ */
+public class QuantileUtil
+{
+ public static ArrayList<Double> getNQuantiles(int numQuantiles)
+ {
+ ArrayList<Double> quantiles = new ArrayList<Double>(numQuantiles);
+ quantiles = new ArrayList<Double>(numQuantiles);
+ int divisor = numQuantiles-1;
+ for (int q = 0; q <= divisor; q++)
+ {
+ double quantile = ((double)q)/divisor;
+ quantiles.add(quantile);
+ }
+ return quantiles;
+ }
+
+ public static ArrayList<Double> getQuantilesFromParams(String... k)
+ {
+ ArrayList<Double> quantiles = new ArrayList<Double>(k.length);
+ for (String s : k) {
+ quantiles.add(Double.parseDouble(s));
+ }
+
+ if (quantiles.size() == 1 && quantiles.get(0) > 1.0)
+ {
+ int numQuantiles = Integer.parseInt(k[0]);
+ if (numQuantiles < 1)
+ {
+ throw new IllegalArgumentException("Number of quantiles must be greater than 1");
+ }
+
+ quantiles = getNQuantiles(numQuantiles);
+ }
+ else
+ {
+ for (Double d : quantiles)
+ {
+ if (d < 0.0 || d > 1.0)
+ {
+ throw new IllegalArgumentException("Quantile must be between 0.0 and 1.0");
+ }
+ }
+ }
+
+ return quantiles;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/778bef1e/datafu-pig/src/main/java/datafu/pig/stats/StreamingMedian.java
----------------------------------------------------------------------
diff --git a/datafu-pig/src/main/java/datafu/pig/stats/StreamingMedian.java b/datafu-pig/src/main/java/datafu/pig/stats/StreamingMedian.java
new file mode 100644
index 0000000..c4c3be4
--- /dev/null
+++ b/datafu-pig/src/main/java/datafu/pig/stats/StreamingMedian.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datafu.pig.stats;
+
+/**
+ * Computes the approximate {@link <a href="http://en.wikipedia.org/wiki/Median" target="_blank">median</a>}
+ * for a (not necessarily sorted) input bag, using the Munro-Paterson algorithm.
+ * This is a convenience wrapper around StreamingQuantile.
+ *
+ * <p>
+ * N.B., all the data is pushed to a single reducer per key, so make sure some partitioning is
+ * done (e.g., group by 'day') if the data is too large. That is, this isn't distributed median.
+ * </p>
+ *
+ * @see StreamingQuantile
+ */
+public class StreamingMedian extends StreamingQuantile
+{
+ public StreamingMedian()
+ {
+ super("0.5");
+ }
+}