You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2009/08/12 08:42:44 UTC
svn commit: r803377 - in /hadoop/pig/trunk: ./
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/
contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/string/
Author: daijy
Date: Wed Aug 12 06:42:43 2009
New Revision: 803377
URL: http://svn.apache.org/viewvc?rev=803377&view=rev
Log:
PIG-907: Provide multiple version of HashFNV (Piggybank)
Added:
hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/HashFNV1.java
hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/HashFNV2.java
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/HashFNV.java
hadoop/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/string/TestHashFNV.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=803377&r1=803376&r2=803377&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Wed Aug 12 06:42:43 2009
@@ -49,6 +49,8 @@
BUG FIXES
+ PIG-907: Provide multiple version of HashFNV (Piggybank) (daijy)
+
PIG-905: TOKENIZE throws exception on null data (daijy)
PIG-901: InputSplit (SliceWrapper) created by Pig is big in size due to
Modified: hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/HashFNV.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/HashFNV.java?rev=803377&r1=803376&r2=803377&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/HashFNV.java (original)
+++ hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/HashFNV.java Wed Aug 12 06:42:43 2009
@@ -18,11 +18,14 @@
package org.apache.pig.piggybank.evaluation.string;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.pig.EvalFunc;
-import org.apache.pig.PigWarning;
+import org.apache.pig.FuncSpec;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
/**
@@ -35,7 +38,6 @@
public class HashFNV extends EvalFunc<Long> {
static final int FNV1_32_INIT = 33554467;
static final int FNV_32_PRIME = 0x01000193;
- int mMod=-1;
public Schema outputSchema(Schema input) {
try {
return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input), DataType.LONG));
@@ -63,31 +65,19 @@
{
return hashFnv32Init(FNV1_32_INIT, s);
}
-
+
+ public Long exec(Tuple input) throws IOException { throw new IOException("HashFNV: internal error, try to use HashFNV directly"); }
+
@Override
- public Long exec(Tuple input) throws IOException {
- if (input.size()!=1 && input.size()!=2) {
- String msg = "HashFNV : Only 1 or 2 parameters are allowed.";
- throw new IOException(msg);
- }
- if (input.get(0)==null)
- return null;
- if (input.size() == 2)
- {
- try {
- mMod = (Integer)input.get(1);
- } catch (ClassCastException e) {
- throw new IOException("HashFNV : 2nd parameter is not Integer",e);
- }
- }
-
- long v = hashFnv32((String)input.get(0));
- if (v < 0)
- v = -v;
- if (mMod > 0)
- {
- v %= mMod;
- }
- return v;
- }
+ public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
+ List<FuncSpec> funcList = new ArrayList<FuncSpec>();
+ Schema s = new Schema();
+ s.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
+ funcList.add(new FuncSpec(HashFNV1.class.getName(), s));
+ s = new Schema();
+ s.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
+ s.add(new Schema.FieldSchema(null, DataType.INTEGER));
+ funcList.add(new FuncSpec(HashFNV2.class.getName(), s));
+ return funcList;
+ }
}
Added: hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/HashFNV1.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/HashFNV1.java?rev=803377&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/HashFNV1.java (added)
+++ hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/HashFNV1.java Wed Aug 12 06:42:43 2009
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.piggybank.evaluation.string;
+
+import java.io.IOException;
+import org.apache.pig.data.Tuple;
+
+/**
+* <dl>
+* Implementation for HashFNV which takes 1 parameter
+* </dl>
+*/
+
+public class HashFNV1 extends HashFNV {
+ @Override
+ public Long exec(Tuple input) throws IOException {
+ if (input.size()!=1) {
+ String msg = "HashFNV : Only 1 parameters are allowed.";
+ throw new IOException(msg);
+ }
+ if (input.get(0)==null)
+ return null;
+
+ long v = hashFnv32((String)input.get(0));
+ if (v < 0)
+ v = -v;
+ return v;
+ }
+}
Added: hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/HashFNV2.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/HashFNV2.java?rev=803377&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/HashFNV2.java (added)
+++ hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/HashFNV2.java Wed Aug 12 06:42:43 2009
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.piggybank.evaluation.string;
+
+import java.io.IOException;
+
+import org.apache.pig.data.Tuple;
+
+/**
+* <dl>
+* Implementation for HashFNV which takes 2 parameter
+* </dl>
+*/
+public class HashFNV2 extends HashFNV {
+ int mMod=-1;
+ @Override
+ public Long exec(Tuple input) throws IOException {
+ if (input.size()!=2) {
+ String msg = "HashFNV : Only 2 parameters are allowed.";
+ throw new IOException(msg);
+ }
+ if (input.get(0)==null)
+ return null;
+ try {
+ mMod = (Integer)input.get(1);
+ } catch (ClassCastException e) {
+ throw new IOException("HashFNV : 2nd parameter is not Integer",e);
+ }
+
+ long v = hashFnv32((String)input.get(0));
+ if (v < 0)
+ v = -v;
+ if (mMod > 0)
+ {
+ v %= mMod;
+ }
+ return v;
+ }
+}
Modified: hadoop/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/string/TestHashFNV.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/string/TestHashFNV.java?rev=803377&r1=803376&r2=803377&view=diff
==============================================================================
--- hadoop/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/string/TestHashFNV.java (original)
+++ hadoop/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/string/TestHashFNV.java Wed Aug 12 06:42:43 2009
@@ -19,7 +19,8 @@
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
-import org.apache.pig.piggybank.evaluation.string.HashFNV;
+import org.apache.pig.piggybank.evaluation.string.HashFNV1;
+import org.apache.pig.piggybank.evaluation.string.HashFNV2;
import org.junit.Test;
import junit.framework.TestCase;
@@ -39,12 +40,19 @@
t3.set(0, null);
t3.set(1, 100);
- HashFNV func = new HashFNV();
- Long r = func.exec(t1);
+ Tuple t4 = TupleFactory.getInstance().newTuple(1);
+ t4.set(0, "024ulhl0dq1tl&b=2");
+
+ HashFNV2 func2 = new HashFNV2();
+ Long r = func2.exec(t1);
assertTrue(r==6228);
- r = func.exec(t2);
+ r = func2.exec(t2);
assertTrue(r==31);
- r = func.exec(t3);
+ r = func2.exec(t3);
assertTrue(r==null);
+
+ HashFNV1 func1 = new HashFNV1();
+ r = func1.exec(t4);
+ assertTrue(r==1669505231);
}
}