You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by pr...@apache.org on 2009/12/17 19:36:36 UTC
svn commit: r891828 [9/15] - in /hadoop/pig/branches/load-store-redesign: ./
contrib/
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/
contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/string/
contrib/z...
Modified: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java (original)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java Thu Dec 17 18:36:28 2009
@@ -91,26 +91,26 @@
System.setProperty("group", "users");
}
group = System.getProperty("group");
- System.out.println("user:" + user + " group: " + group);
- STR_STORAGE = "[s1, s2] COMPRESS BY gz SECURE BY user:"
+ System.out.println("uid:" + user + " gid: " + group);
+ STR_STORAGE = "[s1, s2] COMPRESS BY gz SECURE BY uid:"
+ user
- + " group:"
+ + " gid:"
+ group
- + " perm:777 SERIALIZE BY pig; [m1#{a}] SERIALIZE BY pig COMPRESS BY gz SECURE BY user:"
+ + " perm:777 SERIALIZE BY pig; [m1#{a}] SERIALIZE BY pig COMPRESS BY gz SECURE BY uid:"
+ user
- + " group:"
+ + " gid:"
+ group
- + " perm:777 ; [r1.f1] SECURE BY user:"
+ + " perm:777 ; [r1.f1] SECURE BY uid:"
+ user
- + " group:"
+ + " gid:"
+ group
- + " perm:777 SERIALIZE BY pig COMPRESS BY gz ; [s3, s4, r2.r3.f3] SERIALIZE BY pig SECURE BY user:"
+ + " perm:777 SERIALIZE BY pig COMPRESS BY gz ; [s3, s4, r2.r3.f3] SERIALIZE BY pig SECURE BY uid:"
+ user
- + " group: "
+ + " gid: "
+ group
- + " perm:777 COMPRESS BY gz ; [s5, s6, m2#{x|y}] compREss by gz secURe by user:"
+ + " perm:777 COMPRESS BY gz ; [s5, s6, m2#{x|y}] compREss by gz secURe by uid:"
+ user
- + " group:"
+ + " gid:"
+ group
+ " perm:777 SerialIZE BY pig; [r1.f2, m1#{b}]; [r2.r3.f4, m2#{z}] SERIALIZE BY avro;[s7,s8] SERIALIZE BY AVRO;[s9,s10] COMPRESS BY gz ";
System.out.println("storage: " + STR_STORAGE);
@@ -118,7 +118,7 @@
conf.setInt("table.input.split.minSize", 64 * 1024);
conf.set("table.output.tfile.compression", "none");
// String STR_STORAGE1 =
- // "[s1, s2] COMPRESS BY gz SECURE BY user:"+USER1.getUserName()+" group:"+USER1.getGroupNames()[0]+" perm:777 SERIALIZE BY pig";
+ // "[s1, s2] COMPRESS BY gz SECURE BY uid:"+USER1.getUserName()+" gid:"+USER1.getGroupNames()[0]+" perm:777 SERIALIZE BY pig";
RawLocalFileSystem rawLFS = new RawLocalFileSystem();
fs = new LocalFileSystem(rawLFS);
path = new Path(fs.getWorkingDirectory(), "TestStorageGrammar");
@@ -326,7 +326,7 @@
public void test1() throws IOException, ParseException {
String schema = "s1:string, s2:string";
- String storage = "[s1, s2]COMPRESS BY gz SECURE BY user:user1 group:users perm:744 SERIALIZE BY pig";
+ String storage = "[s1, s2]COMPRESS BY gz SECURE BY uid:user1 gid:users perm:744 SERIALIZE BY pig";
RawLocalFileSystem rawLFS = new RawLocalFileSystem();
fs = new LocalFileSystem(rawLFS);
Path path1 = new Path(path.toString() + "1");
@@ -350,8 +350,8 @@
@Test
public void test2() throws IOException, ParseException {
String schema = "m1:map(string),m2:map(map(int))";
- String storage = "[m1#{a}] SERIALIZE BY pig COMPRESS BY gz SECURE BY user:"
- + user + " group:" + group + " perm:770 ";
+ String storage = "[m1#{a}] SERIALIZE BY pig COMPRESS BY gz SECURE BY uid:"
+ + user + " gid:" + group + " perm:770 ";
Path path1 = new Path(path.toString() + "2");
BasicTable.Writer writer = null;
try {
@@ -373,7 +373,7 @@
@Test
public void test3() throws IOException, ParseException {
String schema = "r1:record(f1:int, f2:long)";
- String storage = "[r1.f1] SECURE BY user:" + user + " group:" + group
+ String storage = "[r1.f1] SECURE BY uid:" + user + " gid:" + group
+ " perm:777 SERIALIZE BY pig COMPRESS BY gz";
Path path1 = new Path(path.toString() + "3");
BasicTable.Writer writer = null;
@@ -395,8 +395,8 @@
@Test
public void test4() throws IOException, ParseException {
String schema = "s3:string, s4:string, r2:record(r3:record(f3:float, f4)), c:collection(f13:double, f14:float, f15:bytes)";
- String storage = "[s3, s4, r2.r3.f3, c] SERIALIZE BY pig SECURE BY user:"
- + user + " group:" + group + " perm:777 COMPRESS BY gz";
+ String storage = "[s3, s4, r2.r3.f3, c] SERIALIZE BY pig SECURE BY uid:"
+ + user + " gid:" + group + " perm:777 COMPRESS BY gz";
Path path1 = new Path(path.toString() + "4");
BasicTable.Writer writer = null;
try {
@@ -417,8 +417,8 @@
@Test
public void test5() throws IOException, ParseException {
String schema = "s5:string, s6:string, m2:map(map(int))";
- String storage = "[s5, s6, m2#{x|y}] compREss by gz secURe by user:" + user
- + " group:" + group + " perm:777 SerialIZE BY pig";
+ String storage = "[s5, s6, m2#{x|y}] compREss by gz secURe by uid:" + user
+ + " gid:" + group + " perm:777 SerialIZE BY pig";
Path path1 = new Path(path.toString() + "5");
BasicTable.Writer writer = null;
try {
@@ -541,7 +541,7 @@
@Test(expected = IOException.class)
public void test12() throws IOException, ParseException {
String schema = "some1:string, some2:string";
- String storage = "[some1,some2] COMPRESS BY gz SECURrrrrrE BY user:user1 group:grop1 perm:760 SERIALIZE BY pig";
+ String storage = "[some1,some2] COMPRESS BY gz SECURrrrrrE BY uid:user1 gid:grop1 perm:760 SERIALIZE BY pig";
Path path1 = new Path(path.toString() + "12");
BasicTable.Writer writer = null;
@@ -553,7 +553,7 @@
@Test(expected = IOException.class)
public void test13() throws IOException, ParseException {
String schema = "some1:string, some2:string";
- String storage = "[some1,some2] COMPRESS gz SECURE BY user:user1 group:users perm:760 SERIALIZE BY pig";
+ String storage = "[some1,some2] COMPRESS gz SECURE BY uid:user1 gid:users perm:760 SERIALIZE BY pig";
Path path1 = new Path(path.toString() + "13");
BasicTable.Writer writer = null;
@@ -565,7 +565,7 @@
@Test(expected = IOException.class)
public void test14() throws IOException, ParseException {
String schema = "some1:string, some2:string";
- String storage = "[some1,some2] COMPRESS BY gz SECURE BY usssser:user1 group:users perm:760 SERIALIZE BY pig";
+ String storage = "[some1,some2] COMPRESS BY gz SECURE BY usssser:user1 gid:users perm:760 SERIALIZE BY pig";
Path path1 = new Path(path.toString() + "14");
BasicTable.Writer writer = null;
writer = new BasicTable.Writer(path1, schema, storage, conf);
@@ -576,7 +576,7 @@
@Test(expected = IOException.class)
public void test15() throws IOException, ParseException {
String schema = "some1:string, some2:string";
- String storage = "[some1,some2] COMPRESS BY gz SECURE BY user:user1 grouuuup:group1 perm:760 SERIALIZE BY pig";
+ String storage = "[some1,some2] COMPRESS BY gz SECURE BY uid:user1 grouuuup:group1 perm:760 SERIALIZE BY pig";
Path path1 = new Path(path.toString() + "15");
BasicTable.Writer writer = null;
writer = new BasicTable.Writer(path1, schema, storage, conf);
@@ -587,7 +587,7 @@
@Test(expected = IOException.class)
public void test16() throws IOException, ParseException {
String schema = "some1:string, some2:string";
- String storage = "[some1,some2] COMPRESS BY gz SECURE BY user:user1 group:users perrrrrm:760 SERIALIZE BY pig";
+ String storage = "[some1,some2] COMPRESS BY gz SECURE BY uid:user1 gid:users perrrrrm:760 SERIALIZE BY pig";
Path path1 = new Path(path.toString() + "16");
BasicTable.Writer writer = null;
writer = new BasicTable.Writer(path1, schema, storage, conf);
@@ -658,7 +658,7 @@
*/
public void test20() throws IOException, ParseException {
String schema = "s9:string, s10:string, s11:string";
- String storage = "[s9,s10] COMPRESS BY gZ; secure by user:user1 group:users perm:755";
+ String storage = "[s9,s10] COMPRESS BY gZ; secure by uid:user1 gid:users perm:755";
Path path1 = new Path(path.toString() + "20");
Runtime.getRuntime().exec("rm -rf " + path1.toString());
@@ -681,7 +681,7 @@
@Test(expected = IOException.class)
public void test21() throws IOException, ParseException {
String schema = "s9:string, s10:string, s11:string";
- String storage = "[s9,s10] COMPRESS BY gZ; secure by user:user1 group:users perm:755;secure by user:user1 group:users perm:755";
+ String storage = "[s9,s10] COMPRESS BY gZ; secure by uid:user1 gid:users perm:755;secure by user:user1 gid:users perm:755";
Path path1 = new Path(path.toString() + "16");
BasicTable.Writer writer = null;
writer = new BasicTable.Writer(path1, schema, storage, conf);
@@ -692,7 +692,7 @@
@Test
public void test22() throws IOException, ParseException {
String schema = "s9:string, s10:string";
- String storage = "[s9,s10] secure by user:user3";
+ String storage = "[s9,s10] secure by uid:user3";
Path path1 = new Path(path.toString() + "22");
Runtime.getRuntime().exec("rm -rf " + path1.toString());
@@ -720,7 +720,7 @@
*/
public void test23() throws IOException, ParseException {
String schema = "s9:string, s10:string";
- String storage = "[s9,s10] secure by group:users";
+ String storage = "[s9,s10] secure by gid:users";
Path path1 = new Path(path.toString() + "23");
Runtime.getRuntime().exec("rm -rf " + path1.toString());
@@ -773,7 +773,7 @@
*/
public void test25() throws IOException, ParseException {
String schema = "s9:string, s10:string";
- String storage = "[s9,s10] secure by user:user1 group:users";
+ String storage = "[s9,s10] secure by uid:user1 gid:users";
Path path1 = new Path(path.toString() + "25");
Runtime.getRuntime().exec("rm -rf " + path1.toString());
@@ -798,7 +798,7 @@
//@Test
public void test26() throws IOException, ParseException {
String schema = "s9:string, s10:string";
- String storage = "[s9,s10] secure by perm:755 group:users";
+ String storage = "[s9,s10] secure by perm:755 gid:users";
Path path1 = new Path(path.toString() + "26");
Runtime.getRuntime().exec("rm -rf " + path1.toString());
Modified: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc1.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc1.java?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc1.java (original)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc1.java Thu Dec 17 18:36:28 2009
@@ -81,13 +81,13 @@
@Test
public void testStorageValid1() {
try {
- String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gz SECURE BY user:root; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
-// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY user:root; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
-// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY user:root group:data perm:0766; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
-// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY user:root group:data perm:966; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
+ String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gz SECURE BY uid:root; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
+// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY uid:root; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
+// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY uid:root group:data perm:0766; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
+// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY uid:root group:data perm:966; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
-// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY user:ggg SECURE BY group:fff; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
-// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY user:root user:root; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
+// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY uid:ggg SECURE BY group:fff; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
+// String strStorage = "[r.r.f1,r.f2#{k1}] COMPRESS BY gzip SECURE BY uid:root user:root; [r.r.f2, r.f2#{k2}] COMPRESS BY lzo SERIALIZE BY avro";
Partition p = new Partition(schema.toString(), strStorage, null);
CGSchema[] cgschemas = p.getCGSchemas();
@@ -161,6 +161,7 @@
}
}
} catch (Exception e) {
+ System.out.println("Error is [" + e.getMessage() + "]");
Assert.assertTrue(false);
}
}
Added: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestZebraTupleTostring.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestZebraTupleTostring.java?rev=891828&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestZebraTupleTostring.java (added)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestZebraTupleTostring.java Thu Dec 17 18:36:28 2009
@@ -0,0 +1,256 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.zebra.types;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import junit.framework.Assert;
+import org.apache.hadoop.zebra.parser.ParseException;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.Tuple;
+import org.junit.Test;
+
+/**
+ *
+ * Test toString() methold of ZebraTuple.
+ * It should generate string representations of Zebra tuples of CSV format.
+ *
+ */
+public class TestZebraTupleTostring {
+ @Test
+ public void testSimple() throws IOException, ParseException {
+ String STR_SCHEMA = "s1:bool, s2:int, s3:long, s4:float, s5:string, s6:bytes";
+ Schema schema = new Schema(STR_SCHEMA);
+ Tuple tuple = TypesUtils.createTuple(schema);
+ Assert.assertTrue(tuple instanceof ZebraTuple);
+ TypesUtils.resetTuple(tuple);
+
+ tuple.set(0, true); // bool
+ tuple.set(1, 1); // int
+ tuple.set(2, 1001L); // long
+ tuple.set(3, 1.1); // float
+ tuple.set(4, "hello\r world, 1\n"); // string
+ tuple.set(5, new DataByteArray("hello byte, 1")); // bytes
+
+ String str = tuple.toString();
+ Assert.assertTrue(str.equals("T,1,1001,1.1,'hello%0D world%2C 1%0A,#hello byte, 1"));
+ }
+
+ @Test
+ public void testRecord() throws IOException, ParseException {
+ String STR_SCHEMA = "r1:record(f1:int, f2:long), r2:record(r3:record(f3:float, f4))";
+ Schema schema = new Schema(STR_SCHEMA);
+ Tuple tuple = TypesUtils.createTuple(schema);
+ TypesUtils.resetTuple(tuple);
+
+ Tuple tupRecord1;
+ Schema r1Schema = new Schema("f1:int, f2:long");
+ tupRecord1 = TypesUtils.createTuple(r1Schema);
+ TypesUtils.resetTuple(tupRecord1);
+
+ Tuple tupRecord2;
+ Schema r2Schema = new Schema("r3:record(f3:float, f4)");
+ tupRecord2 = TypesUtils.createTuple(r2Schema);
+ TypesUtils.resetTuple(tupRecord2);
+
+ Tuple tupRecord3;
+ Schema r3Schema = new Schema("f3:float, f4");
+ tupRecord3 = TypesUtils.createTuple(r3Schema);
+ TypesUtils.resetTuple(tupRecord3);
+
+ // r1:record(f1:int, f2:long)
+ tupRecord1.set(0, 1);
+ tupRecord1.set(1, 1001L);
+ Assert.assertTrue(tupRecord1.toString().equals("1,1001"));
+ tuple.set(0, tupRecord1);
+
+ // r2:record(r3:record(f3:float, f4))
+ tupRecord2.set(0, tupRecord3);
+ tupRecord3.set(0, 1.3);
+ tupRecord3.set(1, new DataByteArray("r3 row1 byte array"));
+ Assert.assertTrue(tupRecord3.toString().equals("1.3,#r3 row1 byte array"));
+ Assert.assertTrue(tupRecord2.toString().equals("1.3,#r3 row1 byte array"));
+ tuple.set(1, tupRecord2);
+ Assert.assertTrue(tuple.toString().equals("1,1001,1.3,#r3 row1 byte array"));
+ }
+
+ @Test
+ public void testMap() throws IOException, ParseException {
+ String STR_SCHEMA = "m1:map(string),m2:map(map(int))";
+ Schema schema = new Schema(STR_SCHEMA);
+ Tuple tuple = TypesUtils.createTuple(schema);
+ TypesUtils.resetTuple(tuple);
+
+ // m1:map(string)
+ Map<String, String> m1 = new HashMap<String, String>();
+ m1.put("a", "'A");
+ m1.put("b", "B,B");
+ m1.put("c", "C\n");
+ tuple.set(0, m1);
+
+ // m2:map(map(int))
+ HashMap<String, Map<String, Integer>> m2 = new HashMap<String, Map<String, Integer>>();
+ Map<String, Integer> m3 = new HashMap<String, Integer>();
+ m3.put("m311", 311);
+ m3.put("m321", 321);
+ m3.put("m331", 331);
+ Map<String, Integer> m4 = new HashMap<String, Integer>();
+ m4.put("m411", 411);
+ m4.put("m421", 421);
+ m4.put("m431", 431);
+ m2.put("x", m3);
+ m2.put("y", m4);
+ tuple.set(1, m2);
+
+ Assert.assertTrue(tuple.toString().equals(
+ "m{'b,'B%2CB,'c,'C%0A,'a,''A},m{'y,m{'m421,421,'m411,411,'m431,431},'x,m{'m311,311,'m321,321,'m331,331}}"));
+ }
+
+ @Test
+ public void testCollection() throws IOException, ParseException {
+ String STR_SCHEMA =
+ "c1:collection(a:double, b:float, c:bytes),c2:collection(r1:record(f1:int, f2:string), d:string),c3:collection(c3_1:collection(e:int,f:bool))";
+ Schema schema = new Schema(STR_SCHEMA);
+ Tuple tuple = TypesUtils.createTuple(schema);
+ TypesUtils.resetTuple(tuple);
+
+ DataBag bag1 = TypesUtils.createBag();
+ Schema schColl = new Schema("a:double, b:float, c:bytes");
+ Tuple tupColl1 = TypesUtils.createTuple(schColl);
+ Tuple tupColl2 = TypesUtils.createTuple(schColl);
+
+ DataBag bag2 = TypesUtils.createBag();
+ Schema schColl2 = new Schema("r1:record(f1:int, f2:string), d:string");
+ Tuple tupColl2_1 = TypesUtils.createTuple(schColl2);
+ Tuple tupColl2_2 = TypesUtils.createTuple(schColl2);
+
+ Tuple collRecord1;
+ try {
+ collRecord1 = TypesUtils.createTuple(new Schema("f1:int, f2:string"));
+ } catch (ParseException e) {
+ e.printStackTrace();
+ throw new IOException(e);
+ }
+ Tuple collRecord2;
+ try {
+ collRecord2 = TypesUtils.createTuple(new Schema("f1:int, f2:string"));
+ } catch (ParseException e) {
+ e.printStackTrace();
+ throw new IOException(e);
+ }
+
+ // c3:collection(c3_1:collection(e:int,f:bool))
+ DataBag bag3 = TypesUtils.createBag();
+ DataBag bag3_1 = TypesUtils.createBag();
+ DataBag bag3_2 = TypesUtils.createBag();
+
+ Tuple tupColl3_1 = null;
+ try {
+ tupColl3_1 = TypesUtils.createTuple(new Schema("e:int,f:bool"));
+ } catch (ParseException e) {
+ e.printStackTrace();
+ throw new IOException(e);
+ }
+ Tuple tupColl3_2;
+ try {
+ tupColl3_2 = TypesUtils.createTuple(new Schema("e:int,f:bool"));
+ } catch (ParseException e) {
+ e.printStackTrace();
+ throw new IOException(e);
+ }
+
+ Tuple tupColl3_3 = null;
+ try {
+ tupColl3_3 = TypesUtils.createTuple(new Schema("e:int,f:bool"));
+ } catch (ParseException e) {
+ e.printStackTrace();
+ throw new IOException(e);
+ }
+ Tuple tupColl3_4;
+ try {
+ tupColl3_4 = TypesUtils.createTuple(new Schema("e:int,f:bool"));
+ } catch (ParseException e) {
+ e.printStackTrace();
+ throw new IOException(e);
+ }
+
+ byte[] abs1 = new byte[3];
+ byte[] abs2 = new byte[4];
+ tupColl1.set(0, 3.1415926);
+ tupColl1.set(1, 1.6);
+ abs1[0] = 'a';
+ abs1[1] = 'a';
+ abs1[2] = 'a';
+ tupColl1.set(2, new DataByteArray(abs1));
+ bag1.add(tupColl1);
+ tupColl2.set(0, 123.456789);
+ tupColl2.set(1, 100);
+ abs2[0] = 'b';
+ abs2[1] = 'c';
+ abs2[2] = 'd';
+ abs2[3] = 'e';
+ tupColl2.set(2, new DataByteArray(abs2));
+ bag1.add(tupColl2);
+ tuple.set(0, bag1);
+
+ collRecord1.set(0, 1);
+ collRecord1.set(1, "record1_string1");
+ tupColl2_1.set(0, collRecord1);
+ tupColl2_1.set(1, "hello1");
+ bag2.add(tupColl2_1);
+
+ collRecord2.set(0, 2);
+ collRecord2.set(1, "record2_string1");
+ tupColl2_2.set(0, collRecord2);
+ tupColl2_2.set(1, "hello2");
+ bag2.add(tupColl2_2);
+ tuple.set(1, bag2);
+
+ TypesUtils.resetTuple(tupColl3_1);
+ TypesUtils.resetTuple(tupColl3_2);
+ tupColl3_1.set(0, 1);
+ tupColl3_1.set(1, true);
+ tupColl3_2.set(0, 2);
+ tupColl3_2.set(1, false);
+ bag3_1.add(tupColl3_1);
+ bag3_1.add(tupColl3_2);
+ bag3.addAll(bag3_1);
+
+ tupColl3_3.set(0, 3);
+ tupColl3_3.set(1, true);
+ tupColl3_4.set(0, 4);
+ tupColl3_4.set(1, false);
+ bag3_2.add(tupColl3_3);
+ bag3_2.add(tupColl3_4);
+ bag3.addAll(bag3_2);
+ tuple.set(2, bag3);
+
+ Assert.assertTrue(tuple.toString().equals("3.1415926,1.6,#aaa\n" +
+ "123.456789,100,#bcde\n" +
+ "1,'record1_string1,'hello1\n"+
+ "2,'record2_string1,'hello2\n"+
+ "1,T\n"+
+ "2,F\n"+
+ "3,T\n"+
+ "4,F\n"));
+ }
+}
+
Modified: hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/cookbook.xml
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/cookbook.xml?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/cookbook.xml (original)
+++ hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/cookbook.xml Thu Dec 17 18:36:28 2009
@@ -64,7 +64,7 @@
<section>
<title>Project Early and Often </title>
-<p>Pig does not (yet) determine when a field is no longer needed and drop the field from the row. For example, say you have a query like: </p>
+<p>Pig does not (yet) determine when a field is no longer needed and drop the field from the row. For example, say you have a query like: </p>
<source>
A = load 'myfile' as (t, u, v);
@@ -74,7 +74,7 @@
E = foreach D generate group, COUNT($1);
</source>
-<p>There is no need for v, y, or z to participate in this query. And there is no need to carry both t and x past the join, just one will suffice. Changing the above query to </p>
+<p>There is no need for v, y, or z to participate in this query. And there is no need to carry both t and x past the join, just one will suffice. Changing the query above to the query below will greatly reduce the amount of data being carried through the map and reduce phases by pig. </p>
<source>
A = load 'myfile' as (t, u, v);
@@ -87,8 +87,7 @@
E = foreach D generate group, COUNT($1);
</source>
-<p>will greatly reduce the amount of data being carried through the map and reduce phases by pig. Depending on your data, this can produce significant time savings. In queries similar to the example given we have seen total time drop by 50%. </p>
-
+<p>Depending on your data, this can produce significant time savings. In queries similar to the example shown here we have seen total time drop by 50%.</p>
</section>
<section>
@@ -147,12 +146,9 @@
</section>
<section>
-<title>Make your UDFs Algebraic</title>
+<title>Make Your UDFs Algebraic</title>
-<p>Queries that can take advantage of the combiner generally ran much faster (sometimes several times faster) than the versions that don't.
-The latest code significantly improves combiner usage; however, you need to make sure you do your part.
-If you have a UDF that works on grouped data and is, by nature, algebraic (meaning their computation can be decomposed into multiple steps)
-make sure you implement it as such. For details on how to write algebraic UDFs, see the <a href="udf.html">Pig UDF Manual</a>. </p>
+<p>Queries that can take advantage of the combiner generally ran much faster (sometimes several times faster) than the versions that don't. The latest code significantly improves combiner usage; however, you need to make sure you do your part. If you have a UDF that works on grouped data and is, by nature, algebraic (meaning their computation can be decomposed into multiple steps) make sure you implement it as such. For details on how to write algebraic UDFs, see the Pig UDF Manual and <a href="udf.html#Aggregate+Functions">Aggregate Functions</a>.</p>
<source>
A = load 'data' as (x, y, z)
@@ -162,12 +158,18 @@
</source>
<p>If <code>MyUDF</code> is algebraic, the query will use combiner and run much faster. You can run <code>explain</code> command on your query to make sure that combiner is used. </p>
+</section>
+<section>
+<title>Implement the Aggregator Interface</title>
+<p>
+If your UDF can't be made Algebraic but is able to deal with getting input in chunks rather than all at once, consider implementing the Aggregator interface to reduce the amount of memory used by your script. If your function <em>is</em> Algebraic and can be used on conjunction with Accumulator functions, you will need to implement the Accumulator interface as well as the Algebraic interface. For more information, see the Pig UDF Manual and <a href="udf.html#Accumulator+Interface">Accumulator Interface</a>.
+</p>
</section>
+
<section>
<title>Drop Nulls Before a Join</title>
-<p>This comment only applies to pig 0.2.0 branch, as pig 0.1.0 does not have nulls. </p>
<p>With the introduction of nulls, join and cogroup semantics were altered to work with nulls. The semantic for cogrouping with nulls is that nulls from a given input are grouped together, but nulls across inputs are not grouped together. This preserves the semantics of grouping (nulls are collected together from a single input to be passed to aggregate functions like COUNT) and the semantics of join (nulls are not joined across inputs). Since flattening an empty bag results in an empty row, in a standard join the rows with a null key will always be dropped. The join: </p>
<source>
@@ -199,10 +201,13 @@
</section>
<section>
-<title>Take Advantage of Join Optimization</title>
+<title>Take Advantage of Join Optimizations</title>
-<p>The optimization insures that the last table in the join is not brought into memory but stream through instead. The optimization reduces the amount of memory used which means you can avoid spilling the data and also should be able to scale your query to larger data volumes. </p>
-<p>To take advantage of this optimization, make sure that the table with the largest number of tuples per key is the last table in your query. </p>
+<section>
+<title>Regular Join Optimizations</title>
+<p>Optimization for regular joins ensures that the last table in the join is not brought into memory but streamed through instead. Optimization reduces the amount of memory used which means you can avoid spilling the data and also should be able to scale your query to larger data volumes. </p>
+<p>To take advantage of this optimization, make sure that the table with the largest number of tuples per key is the last table in your query.
+In some of our tests we saw 10x performance improvement as the result of this optimization.</p>
<source>
small = load 'small_file' as (t, u, v);
@@ -210,34 +215,36 @@
C = join small by t, large by x;
</source>
-<p>In some of our tests we saw 10x performance improvement as the result of this optimization. </p>
</section>
<section>
-<title>Use Fragment Replicate Join</title>
+<title>Specialized Join Optimizations</title>
+<p>Optimization can also be achieved using fragment replicate joins, skewed joins, and merge joins.
+For more information see <a href="piglatin_users.html#Specialized+Joins">Specialized Joins</a>.</p>
+</section>
-<p>This type of join works well if one of tables is small enough to fit into main memory. In this case, Pig can perform a very efficient join since, in hadoop world, it can be done completely on the map side. </p>
+</section>
-<source>
-tiny = load 'small_file' as (t, u, v);
-large = load 'large_file' as (x, y, z);
-C = join large by x, tiny by t using "replicated";
-</source>
-<p>Note that the large table must come first followed by one or more small tables. All small tables together must fit into main memory. </p>
-<p>This feature is new and experimental. It is experimental because we don't have a strong sense of how small the small table must be to fit into memory. In our tests with a simple query that involved just join a table of up to 100M can be used if the process overall gets 1 GB of memory. If the table does not fit into memory, the process would fail and generate an error. </p>
+<section>
+<title>Use the PARALLEL Keyword</title>
-</section>
+<p>PARALLEL controls the number of reducers invoked by Hadoop. The default value is 1. However, the number of reducers you need for a particular construct in Pig that forms a MapReduce boundary depends entirely on (1) your data and the number of intermediate keys you are generating in your mappers and (2) the partitioner and distribution of map (combiner) output keys. In the best cases we have seen that a reducer processing about 500 MB of data behaves efficiently.</p>
-<section>
-<title>Use PARALLEL Keyword</title>
+<p>The keyword makes sense with any operator that starts a reduce phase. This includes
+<a href="piglatin_reference.html#COGROUP">COGROUP</a>,
+<a href="piglatin_reference.html#CROSS">CROSS</a>,
+<a href="piglatin_reference.html#DISTINCT">DISTINCT</a>,
+<a href="piglatin_reference.html#GROUP">GROUP</a>,
+<a href="piglatin_reference.html#JOIN">JOIN</a>,
+<a href="piglatin_reference.html#ORDER">ORDER</a>, and
+<a href="piglatin_reference.html#JOIN%2C+OUTER">OUTER JOIN</a>.
-<p>PARALLEL controls the number of reducers invoked by Hadoop. The default out of the box is 1 which, in most cases, is not what you want. I reasonable heuristic to use is something like </p>
+</p>
-<source><num machines> * <num reduce slots per machine> * 0.9</source>
+<p>You can set the value of PARALLEL in your scripts in conjunction with the operator (see the example below). You can also set the value of PARALLEL for all scripts using the <a href="piglatin_reference.html#set">set</a> command.</p>
-<p>The keyword makes sense on any operator that starts a reduce phase. This includes GROUP, COGROUP, JOIN, DISTINCT, LIMIT, ORDER BY. </p>
-<p>Example: </p>
+<p>Example</p>
<source>
A = load 'myfile' as (t, u, v);
@@ -248,7 +255,7 @@
</section>
<section>
-<title>Use LIMIT</title>
+<title>Use the LIMIT Operator</title>
<p>A lot of the times, you are not interested in the entire output but either a sample or top results. In those cases, using LIMIT can yeild a much better performance as we push the limit as high as possible to minimize the amount of data travelling through the pipeline. </p>
<p>Sample:
Added: hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/images/zebra.jpg
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/images/zebra.jpg?rev=891828&view=auto
==============================================================================
Files hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/images/zebra.jpg (added) and hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/images/zebra.jpg Thu Dec 17 18:36:28 2009 differ
Modified: hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/piglatin_reference.xml
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/piglatin_reference.xml?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/piglatin_reference.xml (original)
+++ hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/piglatin_reference.xml Thu Dec 17 18:36:28 2009
@@ -5451,7 +5451,8 @@
<para>expression</para>
</entry>
<entry>
- <para>A tuple expression. This is the group key or key field. If the result of the tuple expression is a single field, the key will be the value of the first field rather than a tuple with one field.</para>
+ <para>A tuple expression. This is the group key or key field. If the result of the tuple expression is a single field, the key will be the value of the first field rather than a tuple with one field. To group using multiple keys, enclose the keys in parentheses:</para>
+ <para>B = GROUP A BY (key1,key2);</para>
</entry>
</row>
@@ -5657,6 +5658,17 @@
<section>
<title>Example</title>
+<para>This example shows to group using multiple keys.</para>
+<programlisting>
+ A = LOAD 'allresults' USING PigStorage() AS (tcid:int, tpid:int, date:chararray, result:chararray, tsid:int, tag:chararray);
+ B = GROUP A BY (tcid, tpid);
+</programlisting>
+ </section>
+
+
+
+ <section>
+ <title>Example</title>
<para>This example shows a map-side group.</para>
<programlisting>
register zebra.jar;
@@ -5772,7 +5784,9 @@
<section>
<title>Usage</title>
<para>Use the JOIN operator to perform an inner, equijoin join of two or more relations based on common field values.
- The JOIN operator always performs an inner join. Note that the JOIN and COGROUP operators perform similar functions.
+ The JOIN operator always performs an inner join. Inner joins ignore null keys, so it makes sense to filter them out before the join.</para>
+
+ <para>Note that the JOIN and COGROUP operators perform similar functions.
JOIN creates a flat set of output records while COGROUP creates a nested set of output records.</para>
</section>
@@ -9784,6 +9798,17 @@
<entry>
<para>sets user-specified name for the job </para>
</entry>
+ </row>
+ <row>
+ <entry>
+ <para>default_parallel</para>
+ </entry>
+ <entry>
+ <para>whole number </para>
+ </entry>
+ <entry>
+ <para>sets the number of reducers for all MapReduce jobs generated by Pig</para>
+ </entry>
</row></tbody></tgroup>
</informaltable>
<para/>
@@ -9795,6 +9820,7 @@
<programlisting>
grunt> set debug on
grunt> set job.name 'my job'
+grunt> set default_parallel 100
</programlisting>
</section></section>
Modified: hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/piglatin_users.xml
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/piglatin_users.xml?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/piglatin_users.xml (original)
+++ hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/piglatin_users.xml Thu Dec 17 18:36:28 2009
@@ -385,11 +385,14 @@
<section>
<title>Implicit Dependencies</title>
-<p>If a script has dependencies on the execution order outside of what Pig knows about, execution may fail. For instance, in this script
-MYUDF might try to read from out1, a file that A was just stored into.
+<p>If a script has dependencies on the execution order outside of what Pig knows about, execution may fail. </p>
+
+
+<section>
+ <title>Example</title>
+<p>In this script, MYUDF might try to read from out1, a file that A was just stored into.
However, Pig does not know that MYUDF depends on the out1 file and might submit the jobs
-producing the out2 and out1 files at the same time.
-</p>
+producing the out2 and out1 files at the same time.</p>
<source>
...
STORE A INTO 'out1';
@@ -410,6 +413,62 @@
STORE C INTO 'out2';
</source>
</section>
+
+<section>
+ <title>Example</title>
+<p>In this script, the store/load operators have different file paths; however, the load operator depends on the store operator.</p>
+<source>
+A = LOAD '/user/xxx/firstinput' USING PigStorage();
+B = group ....
+C = .... agrregation function
+STORE C INTO '/user/vxj/firstinputtempresult/days1';
+..
+Atab = LOAD '/user/xxx/secondinput' USING PigStorage();
+Btab = group ....
+Ctab = .... agrregation function
+STORE Ctab INTO '/user/vxj/secondinputtempresult/days1';
+..
+E = LOAD '/user/vxj/firstinputtempresult/' USING PigStorage();
+F = group ....
+G = .... aggregation function
+STORE G INTO '/user/vxj/finalresult1';
+
+Etab =LOAD '/user/vxj/secondinputtempresult/' USING PigStorage();
+Ftab = group ....
+Gtab = .... aggregation function
+STORE Gtab INTO '/user/vxj/finalresult2';
+</source>
+
+<p>To make the script works, add the exec statement. </p>
+
+<source>
+A = LOAD '/user/xxx/firstinput' USING PigStorage();
+B = group ....
+C = .... agrregation function
+STORE C INTO '/user/vxj/firstinputtempresult/days1';
+..
+Atab = LOAD '/user/xxx/secondinput' USING PigStorage();
+Btab = group ....
+Ctab = .... agrregation function
+STORE Ctab INTO '/user/vxj/secondinputtempresult/days1';
+
+EXEC;
+
+E = LOAD '/user/vxj/firstinputtempresult/' USING PigStorage();
+F = group ....
+G = .... aggregation function
+STORE G INTO '/user/vxj/finalresult1';
+..
+Etab =LOAD '/user/vxj/secondinputtempresult/' USING PigStorage();
+Ftab = group ....
+Gtab = .... aggregation function
+STORE Gtab INTO '/user/vxj/finalresult2';
+</source>
+
+
+</section>
+</section>
+
</section>
<!-- END MULTI-QUERY EXECUTION-->
@@ -421,7 +480,7 @@
<p>
Pig Latin includes three "specialized" joins: fragement replicate joins, skewed joins, and merge joins.
Replicate, skewed, and merge joins can be performed using the <a href="piglatin_reference.html#JOIN">JOIN</a> operator (inner, equijoins).
-Replicate and skewed joins can be performed using the the the <a href="piglatin_reference.html#JOIN%2C+OUTER">Outer Join</a> syntax.
+Replicate and skewed joins can also be performed using the the the <a href="piglatin_reference.html#JOIN%2C+OUTER">Outer Join</a> syntax.
</p>
<!-- FRAGMENT REPLICATE JOINS-->
@@ -696,77 +755,6 @@
</section> <!-- END MEMORY MANAGEMENT -->
- <!-- ZEBRA INTEGRATION -->
-<section>
-<title>Integration with Zebra</title>
- <p>This version of Pig is integrated with Zebra storage format. Zebra is a recent contrib project of Pig and the details can be found at http://wiki.apache.org/pig/zebra. Pig can now: </p>
- <ul>
- <li>Load data in Zebra format</li>
- <li>Take advantage of sorted Zebra tables in case of map-side group and merge join.</li>
- <li>Store data in Zebra format</li>
- </ul>
- <p></p>
- <p>To load data in Zebra format using TableLoader, do the following:</p>
- <source>
-register /zebra.jar;
-A = LOAD 'studenttab' USING org.apache.hadoop.zebra.pig.TableLoader();
-B = FOREACH A GENERATE name, age, gpa;
-</source>
-
- <p>There are a couple of things to note:</p>
- <ol>
- <li>You need to register a Aebra jar file the same way you would do it for any other UDF.</li>
- <li>You need to place the jar on your classpath.</li>
- <li>Zebra data is self-described and always contains schema. This means that the AS clause is unnecessary as long as
- you know what the column names and types are. To determine the column names and types, you can run the DESCRIBE statement right after the load:
- <source>
-A = LOAD 'studenttab' USING org.apache.hadoop.zebra.pig.TableLoader();
-DESCRIBE A;
-a: {name: chararray,age: int,gpa: float}
-</source>
- </li>
- </ol>
-
-<p>You can provide alternative names to the columns with AS clause. You can also provide types as long as the
- original type can be converted to the new type.</p>
-
-<p>You can provide multiple, comma-separated files to the loader:</p>
-<source>
-A = LOAD 'studenttab, votertab' USING org.apache.hadoop.zebra.pig.TableLoader();
-</source>
-
-<p>TableLoader supports efficient column selection. The current version of Pig does not support automatically pushing
- projections down to the loader. (The work is in progress and will be done after beta.)
- Meanwhile, the loader allows passing columns down via a list of arguments. This example tells the loader to only return two columns, name and age.</p>
-<source>
-A = LOAD 'studenttab' USING org.apache.hadoop.zebra.pig.TableLoader('name, age');
-</source>
-
-<p>If the input data is globally sorted, map-side group or merge join can be used. Please, notice the âsortedâ argument passed to the loader. This lets the loader know that the data is expected to be globally sorted and that a single key must be given to the same map.</p>
-
-<p>Here is an example of the merge join. Note that the first argument to the loader is left empty to indicate that all columns are requested.</p>
-<source>
-A = LOAD'studentsortedtab' USING org.apache.hadoop.zebra.pig.TableLoader('', 'sorted');
-B = LOAD 'votersortedtab' USING org.apache.hadoop.zebra.pig.TableLoader('', 'sorted');
-G = JOIN A BY $0, B By $0 USING "merge";
-</source>
-
-<p>Here is an example of a map-side group. Note that multiple sorted files are passed to the loader and that the loader will perform sort preserving merge to make sure that the data is globally sorted.</p>
-<source>
-A = LOAD 'studentsortedtab, studentnullsortedtab' using org.apache.hadoop.zebra.pig.TableLoader('name, age, gpa, source_table', 'sorted');
-B = GROUP A BY $0 USING "collected";
-C = FOREACH B GENERATE group, MAX(a.$1);
-</source>
-
-<p>You can also write data in Zebra format. Note that, since Zebra requires a schema to be stored with the data, the relation that is stored must have a name assigned (via alias) to every column in the relation.</p>
-<source>
-A = LOAD 'studentsortedtab, studentnullsortedtab' USING org.apache.hadoop.zebra.pig.TableLoader('name, age, gpa, source_table', 'sorted');
-B = GROUP A BY $0 USING "collected";
-C = FOREACH B GENERATE group, MAX(a.$1) AS max_val;
-STORE C INTO 'output' USING org.apache.hadoop.zebra.pig.TableStorer('');
-</source>
-
- </section> <!-- END ZEBRA INTEGRATION -->
Modified: hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/site.xml?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/site.xml (original)
+++ hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/site.xml Thu Dec 17 18:36:28 2009
@@ -50,6 +50,14 @@
<cookbook label="Cookbook" href="cookbook.html" />
<udf label="UDFs" href="udf.html" />
</docs>
+ <docs label="Zebra">
+ <piglatin label="Zebra Overview " href="zebra_overview.html" />
+ <piglatin label="Zebra Users " href="zebra_users.html" />
+ <piglatin label="Zebra Reference " href="zebra_reference.html" />
+ <piglatin label="Zebra MapReduce " href="zebra_mapreduce.html" />
+ <piglatin label="Zebra Pig " href="zebra_pig.html" />
+ <piglatin label="Zebra Streaming " href="zebra_stream.html" />
+ </docs>
<docs label="Miscellaneous">
<api label="API Docs" href="ext:api"/>
<wiki label="Wiki" href="ext:wiki" />
Modified: hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/udf.xml
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/udf.xml?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/udf.xml (original)
+++ hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/udf.xml Thu Dec 17 18:36:28 2009
@@ -342,7 +342,7 @@
</tr>
</table>
-<p>All Pig-specific classes are available <a href="http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/"> here</a> </p>
+<p>All Pig-specific classes are available <a href="http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/"> here</a>. </p>
<p><code>Tuple</code> and <code>DataBag</code> are different in that they are not concrete classes but rather interfaces. This enables users to extend Pig with their own versions of tuples and bags. As a result, UDFs cannot directly instantiate bags or tuples; they need to go through factory classes: <code>TupleFactory</code> and <code>BagFactory</code>. </p>
<p>The builtin <code>TOKENIZE</code> function shows how bags and tuples are created. A function takes a text string as input and returns a bag of words from the text. (Note that currently Pig bags always contain tuples.) </p>
@@ -752,11 +752,11 @@
public Integer bytesToInteger(byte[] b) throws IOException;
public Long bytesToLong(byte[] b) throws IOException;
......
- public void fieldsToRead(Schema schema);
+ public RequiredFieldResponse fieldsToRead(RequiredFieldList requiredFieldList) throws FrontendException;
public Schema determineSchema(String fileName, ExecType execType, DataStorage storage) throws IOException;
</source>
-
+<p><strong>bindTo</strong></p>
<p>The <code>bindTo</code> function is called once by each Pig task before it starts processing data. It is intended to connect the function to its input. It provides the following information: </p>
<ul>
<li><p> <code>fileName</code> - The name of the file from which the data is read. Not used most of the time </p>
@@ -770,7 +770,11 @@
</ul>
<p>In the Hadoop world, the input data is treated as a continuous stream of bytes. A <code>slicer</code>, discussed in the Advanced Topics section, is used to split the data into chunks with each chunk going to a particular task for processing. This chunk is what <code>bindTo</code> provides to the UDF. Note that unless you use a custom slicer, the default slicer is not aware of tuple boundaries. This means that the chunk you get can start and end in the middle of a particular tuple. One common approach is to skip the first partial tuple and continue past the end position to finish processing a tuple. This is what <code>PigStorage</code> does as the example later in this section shows. </p>
+
+<p><strong>getNext</strong></p>
<p>The <code>getNext</code> function reads the input stream and constructs the next tuple. It returns <code>null</code> when it is done with processing and throws an <code>IOException</code> if it fails to process an input tuple. </p>
+
+<p><strong>conversion routines</strong></p>
<p>Next is a bunch of conversion routines that convert data from <code>bytearray</code> to the requested type. This requires further explanation. By default, we would like the loader to do as little per-tuple processing as possible. This is because many tuples can be thrown out during filtering or joins. Also, many fields might not get used because they get projected out. If the data needs to be converted into another form, we would like this conversion to happen as late as possible. The majority of the loaders should return the data as bytearrays and the Pig will request a conversion from bytearray to the actual type when needed. Let's looks at the example below: </p>
<source>
@@ -781,11 +785,32 @@
</source>
<p>In this query, only <code>age</code> needs to be converted to its actual type (=int=) right away. <code>name</code> only needs to be converted in the next step of processing where the data is likely to be much smaller. <code>gpa</code> is not used at all and will never need to be converted. </p>
+
<p>This is the main reason for Pig to separate the reading of the data (which can happen immediately) from the converting of the data (to the right type, which can happen later). For ASCII data, Pig provides <code>Utf8StorageConverter</code> that your loader class can extend and will take care of all the conversion routines. The code for it can be found <a href="http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/builtin/Utf8StorageConverter.java?view=markup"> here</a>. </p>
+
<p>Note that conversion rutines should return null values for data that can't be converted to the specified type. </p>
+
<p>Loaders that work with binary data like <code>BinStorage</code> are not going to use this model. Instead, they will produce objects of the appropriate types. However, they might still need to define conversion routines in case some of the fields in a tuple are of type <code>bytearray</code>. </p>
-<p><code>fieldsToRead</code> is reserved for future use and should be left empty. </p>
+
+<p><strong>fieldsToRead</strong></p>
+<p>
+The intent of the <code>fieldsToRead</code> function is to reduce the amount of data returned from the loader. Pig will evaluate the script and determine the minimal set of columns needed to execute it. This information will be passed to the <code>fieldsToRead</code> function of the loader in the <code>requiredFieldList</code> parameter. The parameter is of type <code>RequiredFieldList</code> that is defined as part of the
+<a href="http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/LoadFunc.java?view=markup">LoadFunc</a> interface.
+If the loader chooses not to purge unneeded columns, it can use the following implementation:
+</p>
+<source>
+public LoadFunc.RequiredFieldResponse fieldsToRead(LoadFunc.RequiredFieldList requiredFieldList) throws FrontendException {
+ return new LoadFunc.RequiredFieldResponse(false);
+}
+</source>
+
+<p>
+This tells Pig that it should expect the entire column set from the loader. We expect that most loaders will stick to this implementation. In our tests of PigStorage, we saw about 5% improvement when selecting 5 columns out of 40. The loaders that should take advantage of this functionality are the ones, like Zebra, that can pass this information directly to the storage layer. For an example of <code>fieldsToRead</code> see the implementation in <a href="http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/src/org/apache/pig/builtin/PigStorage.java?view=markup"> PigStorage</a>.
+</p>
+
+<p><strong>determineSchema</strong></p>
<p>The <code>determineSchema</code> function must be implemented by loaders that return real data types rather than <code>bytearray</code> fields. Other loaders should just return <code>null</code>. The idea here is that Pig needs to know the actual types it will be getting; Pig will call <code>determineSchema</code> on the client side to get this information. The function is provided as a way to sample the data to determine its schema. </p>
+
<p>Here is the example of the function implemented by =BinStorage=: </p>
<source>
@@ -866,7 +891,7 @@
</section>
<section>
-<title>Accumulate Interface</title>
+<title>Accumulator Interface</title>
<p>In Pig, problems with memory usage can occur when data, which results from a group or cogroup operation, needs to be placed in a bag and passed in its entirety to a UDF.</p>
@@ -880,7 +905,7 @@
*/
public void accumulate(Tuple b) throws IOException;
/**
- * Called when all tuples from current key have been passed to accumulate.
+ * Called when all tuples from current key have been passed to the accumulator.
* @return the value for the UDF for this key.
*/
public T getValue();
Added: hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/zebra_mapreduce.xml
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/zebra_mapreduce.xml?rev=891828&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/zebra_mapreduce.xml (added)
+++ hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/zebra_mapreduce.xml Thu Dec 17 18:36:28 2009
@@ -0,0 +1,606 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!-- Copyright 2002-2004 The Apache Software Foundation
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN"
+ "http://forrest.apache.org/dtd/document-v20.dtd">
+
+ <!-- BEGIN DOCUMENT-->
+
+<document>
+<header>
+<title>Zebra and MapReduce</title>
+</header>
+<body>
+
+
+ <!-- OVERVIEW-->
+ <section>
+ <title>Overview</title>
+ <p>MapReduce allows you to take full advantage of Zebra's capabilities.</p>
+ </section>
+ <!-- END OVERVIEW-->
+
+<!-- HADOOP M/R API-->
+ <section>
+ <title>Hadoop MapReduce APIs</title>
+ <p>Zebra requires Hadoop 20. However, this release of Zebra only supports the "old" jobconf-style MapReduce APIs. </p>
+ <ul>
+ <li>"old" mapreduce API - org.apache.hadoop.mapred.* - supported</li>
+ <li>"new" mapreduce API - org.apache.hadoop.mapreduce.* - not supported</li>
+ </ul>
+ <p></p>
+ </section>
+<!-- END HADOOP M/R API-->
+
+
+ <!-- ZEBRA M/R EXAMPLES-->
+<section>
+<title>Zebra MapReduce Examples</title>
+
+
+
+
+ <!-- ZEBRA OUTPUT EXAMPLE-->
+<section>
+<title>Table Output Format</title>
+<p>This MapReduce example demonstrates the Zebra table output format.
+The Zebra table in this example has two unsorted columns groups, each of which has one column.
+The output format is specified as follows:</p>
+
+<source>
+BasicTableOutputFormat.setStorageInfo(jobConf,
+ ZebraSchema.createZebraSchema("word:string, count:int"),
+ ZebraStorageHint.createZebraStorageHint("[word];[count]"),
+ null);
+</source>
+
+<p><br></br>The input file for this example should contain rows of word and count, separated by a space. For example:</p>
+
+<source>
+this 2
+is 1
+a 5
+test 2
+hello 1
+world 3
+</source>
+
+<p>
+The example works like this. The first job is in Zebra format. The second job reads output from the first job, where Count is specified as a projection column.
+The table input format projects an input row which has both Word and Count into a row containing only the Count column and hands it to map.
+The reducer sums the counts and produces a sum of counts which should match total number of words in original text.
+</p>
+
+<source>
+package org.apache.hadoop.zebra.mapred;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.pig.data.Tuple;
+import org.apache.hadoop.zebra.parser.ParseException;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+public class TableMapReduceExample extends Configured implements Tool {
+
+ static class Map extends MapReduceBase implements Mapper<LongWritable, Text, BytesWritable, Tuple> {
+ private BytesWritable bytesKey;
+ private Tuple tupleRow;
+
+ /**
+ * Map method for reading input.
+ */
+ @Override
+ public void map(LongWritable key, Text value,
+ OutputCollector<BytesWritable, Tuple> output, Reporter reporter)
+ throws IOException {
+
+ // value should contain "word count"
+ String[] wordCount = value.toString().split(" ");
+ if (wordCount.length != 2) {
+ // LOG the error
+ throw new IOException("Value does not contain two fields:" + value);
+ }
+
+ byte[] word = wordCount[0].getBytes();
+ bytesKey.set(word, 0, word.length);
+ tupleRow.set(0, new String(word));
+ tupleRow.set(1, Integer.parseInt(wordCount[1]));
+
+ output.collect(bytesKey, tupleRow);
+ }
+
+ /**
+ * Configuration of the job. Here we create an empty Tuple Row.
+ */
+ @Override
+ public void configure(JobConf job) {
+ bytesKey = new BytesWritable();
+ try {
+ Schema outSchema = BasicTableOutputFormat.getSchema(job);
+ tupleRow = TypesUtils.createTuple(outSchema);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ } catch (ParseException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ static class ProjectionMap extends MapReduceBase implements Mapper<BytesWritable, Tuple, Text, IntWritable> {
+ private final static Text all = new Text("All");
+
+ /**
+ * Map method which gets count column after projection.
+ *
+ * @throws IOException
+ */
+ @Override
+ public void map(BytesWritable key, Tuple value,
+ OutputCollector<Text, IntWritable> output, Reporter reporter)
+ throws IOException {
+ output.collect(all, new IntWritable((Integer) value.get(0)));
+ }
+ }
+
+ public static class ProjectionReduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> {
+ /**
+ * Reduce method which implements summation. Acts as both reducer and combiner.
+ *
+ * @throws IOException
+ */
+ public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
+ int sum = 0;
+ while (values.hasNext()) {
+ sum += values.next().get();
+ }
+ output.collect(key, new IntWritable(sum));
+ }
+ }
+
+ /**
+ * Where jobs and their settings and sequence is set.
+ *
+ * @param args arguments with exception of Tools understandable ones.
+ */
+ public int run(String[] args) throws Exception {
+ if (args == null || args.length != 3) {
+ System.out.println("usage: TableMapReduceExample input_path_for_text_file output_path_for_table output_path_for_text_file");
+ System.exit(-1);
+ }
+
+ /*
+ First MR Job creating a Table with two columns
+ */
+ JobConf jobConf = new JobConf();
+ jobConf.setJobName("TableMapReduceExample");
+ jobConf.set("table.output.tfile.compression", "none");
+
+ // Input settings
+ jobConf.setInputFormat(TextInputFormat.class);
+ jobConf.setMapperClass(Map.class);
+ FileInputFormat.setInputPaths(jobConf, new Path(args[0]));
+
+ // Output settings
+ jobConf.setOutputFormat(BasicTableOutputFormat.class);
+ BasicTableOutputFormat.setOutputPath(jobConf, new Path(args[1]));
+
+ // set the output info:
+ // ZebraSchema - 2 logical columns
+ // ZebraStorageHint - 2 physical column groups (one column each)
+ // ZebraSortInfo - unsorted table (null)
+
+ BasicTableOutputFormat.setStorageInfo(jobConf,
+ ZebraSchema.createZebraSchema("word:string, count:int"),
+ ZebraStorageHint.createZebraStorageHint("[word];[count]"),
+ null);
+
+ // set map-only job.
+ jobConf.setNumReduceTasks(0);
+
+ // Run Job
+ JobClient.runJob(jobConf);
+
+ /*
+ Second MR Job for Table Projection of count column
+ */
+ JobConf projectionJobConf = new JobConf();
+ projectionJobConf.setJobName("TableProjectionMapReduceExample");
+
+ // Input settings
+ projectionJobConf.setMapperClass(ProjectionMap.class);
+ projectionJobConf.setInputFormat(TableInputFormat.class);
+ TableInputFormat.setProjection(projectionJobConf, "count");
+ TableInputFormat.setInputPaths(projectionJobConf, new Path(args[1]));
+ projectionJobConf.setMapOutputKeyClass(Text.class);
+ projectionJobConf.setMapOutputValueClass(IntWritable.class);
+
+ // Output settings
+ projectionJobConf.setOutputFormat(TextOutputFormat.class);
+ FileOutputFormat.setOutputPath(projectionJobConf, new Path(args[2]));
+ projectionJobConf.setReducerClass(ProjectionReduce.class);
+ projectionJobConf.setCombinerClass(ProjectionReduce.class);
+
+ // Run Job
+ JobClient.runJob(projectionJobConf);
+
+ return 0;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new TableMapReduceExample(), args);
+ System.exit(res);
+ }
+}
+</source>
+</section>
+ <!-- END ZEBRA TABLE OUTPUT EXAMPLE-->
+
+
+ <!-- ZEBRA OUTPUT EXAMPLE-->
+<section>
+<title>Table Input/Output Formats</title>
+<p>This MapReduce example demonstrates the Zebra table input/output formats. </p>
+ <source>
+ package org.apache.hadoop.zebra.mapred;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.ArrayList;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.zebra.mapred.BasicTableOutputFormat;
+import org.apache.hadoop.zebra.mapred.TableInputFormat;
+import org.apache.hadoop.zebra.parser.ParseException;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.data.Tuple;
+
+/**
+ * This is a sample to show using zebra table to do a simple basic union in
+ * map/reduce * To run this, we need have two basic tables ready. They contain
+ * the data as in Sample 1, i.e., (word, count). In this example, they are at:
+ * /user/mapredu/t1 /user/mapredu/t2 The resulting table is put at: /user/mapredu2/t1
+ *
+ */
+public class TableMRSample2 {
+ static class MapClass implements
+ Mapper<BytesWritable, Tuple, BytesWritable, Tuple> {
+ private BytesWritable bytesKey;
+ private Tuple tupleRow;
+
+ @Override
+ public void map(BytesWritable key, Tuple value,
+ OutputCollector<BytesWritable, Tuple> output, Reporter reporter)
+ throws IOException
+
+ {
+ System.out.println(key.toString() + value.toString());
+ output.collect(key, value);
+ }
+
+ @Override
+ public void configure(JobConf job) {
+ bytesKey = new BytesWritable();
+ try {
+ Schema outSchema = BasicTableOutputFormat.getSchema(job);
+ tupleRow = TypesUtils.createTuple(outSchema);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ } catch (ParseException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ // no-op
+ }
+
+ public static void main(String[] args) throws ParseException, IOException {
+ JobConf jobConf = new JobConf();
+ jobConf.setJobName("tableMRSample");
+ jobConf.set("table.output.tfile.compression", "gz");
+
+ // input settings
+ jobConf.setInputFormat(TableInputFormat.class);
+ jobConf.setOutputFormat(BasicTableOutputFormat.class);
+ jobConf.setMapperClass(TableMRSample2.MapClass.class);
+
+ List
+ <Path> paths = new ArrayList<Path>(2);
+ Path p = new Path("/user/mapredu/t1");
+ System.out.println("path = " + p);
+ paths.add(p);
+ p = new Path("/user/mapredu/t2");
+ paths.add(p);
+
+ TableInputFormat.setInputPaths(jobConf, paths.toArray(new Path[2]));
+ ZebraProjection zebraProjection = ZebraProjection.createZebraProjection("word");
+ TableInputFormat.setProjection(jobConf, zebraProjection);
+ BasicTableOutputFormat.setOutputPath(jobConf, new Path("/user/mapredu2/t1"));
+
+ ZebraSchema zebraSchema = ZebraSchema.createZebraSchema("word:string");
+ ZebraStorageHint zebraStorageHint = ZebraStorageHint.createZebraStorageHint("[word]");
+ BasicTableOutputFormat.setStorageInfo(jobConf, zebraSchema, zebraStorageHint, null);
+
+ // set map-only job.
+ jobConf.setNumReduceTasks(0);
+ jobConf.setNumMapTasks(2);
+ JobClient.runJob(jobConf);
+ }
+ }
+}
+</source>
+ </section>
+ <!-- END ZEBRA OUTPUT EXAMPLE-->
+
+ <!-- ZEBRA SORT EXAMPLE-->
+<section>
+<title>Sort Columns</title>
+<p>This MapReduce code snippet demonstrates how to sort Zebra columns. </p>
+
+<source>
+/* user provides a Comparator Class for creating ZebraSortInfo */
+
+public static final class MemcmpRawComparator implements
+ RawComparator<Object>, Serializable {
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ return WritableComparator.compareBytes(b1, s1, l1, b2, s2, l2);
+ }
+
+ @Override
+ public int compare(Object o1, Object o2) {
+ throw new RuntimeException("Object comparison not supported");
+ }
+}
+
+â¦
+â¦
+
+ZebraSchema zSchema = ZebraSchema.createZebraSchema(schemaString);
+
+ZebraStorageHint zStorageHint = ZebraStorageHint.createZebraStorageHint(storageHintString);
+
+/* Here we can use above Comparator Class to create a ZebraSortInfo object */
+
+ZebraSortInfo zSortInfo = ZebraSortInfo.createZebraSortInfo(sortColumnsString, MemcmpRawComparator.class);
+
+BasicTableOutputFormat.setStorageInfo(jobConf, zSchema, zStorageHint, zSortInfo);
+</source>
+</section>
+ <!-- END ZEBRA SORT EXAMPLE-->
+
+
+ <!-- ZEBRA DROP-CG EXAMPLE-->
+<section>
+<title>Drop Column Groups</title>
+<p>
+This example illustrates how to drop column groups (CG) in Zebra tables.
+This is not a MapReduce program since the API for deleting columg groups is a simple Java API.
+</p>
+
+<p>How to compile:</p>
+<source>
+# this command requires pig.jar and latest zebra jar.
+$ javac -cp pig.jar:zebra-0.1.0.jar DropColumnGroupExample.java
+
+# create a jar file
+$ jar cvf dropcg.jar DropColumnGroupExample.class
+</source>
+
+<p>How to run:</p>
+<source>
+# run the example.
+$ java -cp pig.jar:zebra-0.1.0.jar:dropcg.jar DropColumnGroupExample
+
+# This creates a table under the directory "dropCGExample".
+# If run the same command again, it fails since the destination
+# directory still exists. Please remove the directory before running.
+
+#This program takes one argument : directory for the example table
+$ java -cp pig.jar:zebra-0.1.0.jar:dropcg.jar DropColumnGroupExample tableDir
+</source>
+
+<p>Source code:</p>
+<source>
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.io.TableScanner;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.data.Tuple;
+
+public class DropColumnGroupExample {
+
+ private static String schema = "url, outcount:int, content:string";
+ private static String storageHint = "[url, outcount]; [content] as data";
+
+ public static void main(String[] args) throws Exception {
+
+ // Create configuration and process generaic Hadoop options like -D.
+ Configuration conf = new Configuration();
+ args = new GenericOptionsParser(conf, args).getRemainingArgs();
+
+ Path path = new Path((args.length > 0 ? args[0] : "dropCGExample"));
+
+ if (path.getFileSystem(conf).exists(path)) {
+ throw new IOException("Destination path '" + path + "' already exists. " +
+ "Please remove the directory and run again.");
+ }
+ // create a simple table.
+ createTable(conf, path);
+
+ //verify we can read the table.
+ String content = getContentForUrl(path, conf, "http://www.yahoo.com/");
+ if (content == null || !content.contains("yahoo")) {
+ throw new IOException("Table read failed.");
+ }
+
+ /* Now drop the colum group named "data".
+ *
+ * An exception is thrown if the CG could not be removed for some reason
+ * (e.g. the user might not have enough permissions).
+ */
+ BasicTable.dropColumnGroup(path, conf, "data");
+
+ // deleting an already deleted CG is a no-op.
+ BasicTable.dropColumnGroup(path, conf, "data");
+
+ // now try to read content column.
+ // Note that NULL is returned for the third column.
+ if (getContentForUrl(path, conf, "http://www.yahoo.com/") != null) {
+ throw new IOException("Expected NULL for 3rd column");
+ }
+ // While reading this table, a warning is logged since the user
+ // is trying to reading from a deleted CG.
+
+ //clean up the test directory.
+ //for now we are not deleting the directory to let users check it.
+ //BasicTable.drop(path, conf);
+ }
+
+ // Utility functions:
+
+ /**
+ * This is a simple table reader that iterates over the table to
+ * find a given url and returns its content.
+ */
+ private static String getContentForUrl(Path path, Configuration conf,
+ String url) throws Exception {
+
+ BasicTable.Reader reader = new BasicTable.Reader(path, conf);
+ TableScanner scanner = reader.getScanner(null, true);
+ Tuple row = TypesUtils.createTuple(3);
+
+ try {
+ while (!scanner.atEnd()) {
+ scanner.getValue(row);
+ if (url.equals(row.get(0))) {
+ return (String)row.get(2);
+ }
+ scanner.advance();
+ }
+ } finally {
+ scanner.close();
+ }
+ throw new IOException(url + " is not found");
+ }
+
+ private static void createTable(Configuration conf, Path path)
+ throws Exception {
+ /* NOTE: This creates a table using BasicTable API. This is not
+ * a committed public API yet. Typically tables are created
+ * in M/R or through PIG.
+ */
+ BasicTable.Writer writer = new BasicTable.Writer(path, schema,
+ storageHint, conf);
+ TableInserter inserter = writer.getInserter("part-0", true);
+ Tuple rowTuple = TypesUtils.createTuple(3);
+ BytesWritable emptyKey = new BytesWritable();
+
+ // add two rows:
+ rowTuple.set(0, "http://www.cnn.com/");
+ rowTuple.set(1, 10);
+ rowTuple.set(2, "content for cnn.com");
+
+ inserter.insert(emptyKey, rowTuple);
+
+ rowTuple.set(0, "http://www.yahoo.com/");
+ rowTuple.set(1, 20);
+ rowTuple.set(2, "content for yahoo.com");
+
+ inserter.insert(emptyKey, rowTuple);
+
+ inserter.close();
+ }
+}
+</source>
+</section>
+ <!-- END ZEBRA DROP-CG EXAMPLE-->
+
+ <!-- MULTIPLE TABLE OUTPUTS-->
+<section>
+<title>Multiple Table Outputs</title>
+<p>
+This code snippet illustrates how to work with multiple table outputs.
+</p>
+<source>
+In main()
+
+ String multiLocs = "/user/multi/us" + "," + "/user/multi/india" + "," + "/user/multi/japan";
+
+ jobConf.setOutputFormat(BasicTableOutputFormat.class);
+ BasicTableOutputFormat.setMultipleOutputPaths(jobConf, multiLocs);
+ BasicTableOutputFormat.setZebraOutputPartitionClass(jobConf, MultipleOutputsTest.OutputPartitionerClass.class);
+
+
+Implement a partition class:
+
+ static class OutputPartitionerClass implements ZebraOutputPartition {
+ @Override
+ public int getOutputPartition(BytesWritable key, Tuple value, String commaSeparatedLocs) {
+
+ String reg = null;
+ try {
+ reg = (String)(value.get(0));
+ } catch ( ExecException e) {
+ // do something about e
+ }
+
+ if(reg.equals("us")) return 0;
+ if(reg.equals("india")) return 1;
+ if(reg.equals("japan")) return 2;
+
+ return 0;
+ }
+ }
+</source>
+ </section>
+
+ <!-- endMULTIPLE TABLE OUTPUTS-->
+
+
+ </section>
+ <!--END ZEBRA M/R EXAMPLES-->
+
+ </body>
+ </document>
+
+
Added: hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/zebra_overview.xml
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/zebra_overview.xml?rev=891828&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/zebra_overview.xml (added)
+++ hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/zebra_overview.xml Thu Dec 17 18:36:28 2009
@@ -0,0 +1,93 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!-- Copyright 2002-2004 The Apache Software Foundation
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN"
+ "http://forrest.apache.org/dtd/document-v20.dtd">
+
+ <!-- BEGIN DOCUMENT-->
+
+<document>
+<header>
+<title>Overview</title>
+</header>
+<body>
+
+
+ <!-- OVERVIEW-->
+ <section>
+ <title>About Zebra</title>
+ <p>Zebra is an access path library for reading and writing data in a column-oriented fashion. Zebra functions as an abstraction layer between your client application and data on the Hadoop Distributed File System (HDFS). Data is written to HDFS using Zebraâs TableStore class. Data is read from HDFS using Zebraâs TableLoad class. Zebra supports client applications written as Pig, MapReduce, or streaming jobs. Keep in mind that Zebra works with Zebra tables; you cannot use Zebra to process text or sequence files.</p>
+<figure alt="Zebra" src="images/zebra.jpg" align="left"/>
+ </section>
+ <!-- END OVERVIEW-->
+
+
+ <!-- ZEBRA SETUP-->
+ <section>
+ <title>Zebra Setup</title>
+ <section>
+ <title>Prerequisites</title>
+ <p>Zebra requires:</p>
+ <ul>
+ <li>Pig 0.6.0 </li>
+ <li>Hadoop 0.20.1</li>
+ </ul>
+ <p></p>
+ <p>Also, make sure the following software is installed on your system:</p>
+ <ul>
+ <li>JDK 1.6</li>
+ <li>Ant 1.7.1</li>
+ <li>Javacc 4.2</li>
+ </ul>
+ </section>
+
+ <section>
+ <title>Download Zebra</title>
+<p>Zebra is a Pig contrib project and is available at: <br></br>
+ <a href="http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/">http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/</a></p>
+<p>To work with Zebra you need to check out the Pig trunk: <br></br>
+ <a href="http://svn.apache.org/repos/asf/hadoop/pig/trunk/ ">http://svn.apache.org/repos/asf/hadoop/pig/trunk/</a></p>
+ </section>
+
+ <section>
+ <title>Compile Zebra</title>
+ <p> To compile Zebra follow these steps.</p>
+
+<p>Step 1:</p>
+<ul>
+<li>Move to the top level of your Pig installation</li>
+<li>Run 'ant jar' (this builds the Pig classes and creates the Pig JAR files)</li>
+<li>(optional) Run 'ant -Dtestcase=none test-core' (this builds the Pig test classes which are needed by the Zebra tests)</li>
+</ul>
+ <p></p>
+<p>Step 2:</p>
+<ul>
+<li>cd ./contrib/zebra </li>
+<li>Run âant jarâ (this builds the Zebra classes and creates the Zebra JAR file)</li>
+<li>(optional) Run âant testâ (this verifies that Zebra is working correctly)</li>
+</ul>
+ </section>
+
+ </section>
+ <!-- END ZEBRA SETUP-->
+
+
+
+
+ </body>
+ </document>
+
+
Added: hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/zebra_pig.xml
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/zebra_pig.xml?rev=891828&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/zebra_pig.xml (added)
+++ hadoop/pig/branches/load-store-redesign/src/docs/src/documentation/content/xdocs/zebra_pig.xml Thu Dec 17 18:36:28 2009
@@ -0,0 +1,189 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!-- Copyright 2002-2004 The Apache Software Foundation
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN"
+ "http://forrest.apache.org/dtd/document-v20.dtd">
+
+ <!-- BEGIN DOCUMENT-->
+
+<document>
+<header>
+<title>Zebra and Pig</title>
+</header>
+<body>
+
+ <!-- OVERVIEW-->
+ <section>
+ <title>Overview</title>
+ <p>With Pig you can load and store data in Zebra format. You can also take advantage of sorted Zebra tables for map-side groups and merge joins. When working with Pig keep in mind that, unlike MapReduce, you do not need to declare Zebra schemas. Zebra automatically converts Zebra schemas to Pig schemas (and vice versa) for you.</p>
+
+ </section>
+ <!-- END OVERVIEW-->
+
+
+ <!-- ZEBRA EXAMPLES-->
+ <section>
+ <title>Zebra Pig Examples</title>
+<p></p>
+
+
+ <!--example: loading data-->
+ <section>
+ <title>Loading Data</title>
+ <p>To load data stored in Zebra format (columns) use the TableLoader function.</p>
+ <source>
+register $LOCATION/zebra-$version.jar;
+A = LOAD 'studenttab' USING org.apache.hadoop.zebra.pig.TableLoader();
+B = FOREACH A GENERATE name, age, gpa;
+</source>
+
+ <p>There are a couple of things to note:</p>
+ <ol>
+ <li>You need to register a Zebra jar file the same way you would do it for any other UDF.</li>
+ <li>You need to place the jar on your classpath.</li>
+ <li>When using Zebra with Pig, Zebra data is self-described and always contains a schema. This means that the AS clause is unnecessary as long as
+ you know what the column names and types are. To determine the column names and types, you can run the DESCRIBE statement right after the load:
+ <source>
+A = LOAD 'studenttab' USING org.apache.hadoop.zebra.pig.TableLoader();
+DESCRIBE A;
+a: {name: chararray,age: int,gpa: float}
+</source>
+ </li>
+ </ol>
+
+<p>You can provide alternative names to the columns with the AS clause. You can also provide types as long as the
+ original type can be converted to the new type. <em>In general</em>, Zebra supports Pig type compatibilities
+ (see <a href="piglatin_reference.html#Arithmetic+Operators+and+More">Arithmetic Operators and More</a>).</p>
+
+<p>You can provide multiple, comma-separated files to the loader:</p>
+<source>
+A = LOAD 'studenttab, votertab' USING org.apache.hadoop.zebra.pig.TableLoader();
+</source>
+
+<p>TableLoader supports efficient column selection; projections are automatically push down to the loader. This example tells the loader to only return two columns, name and age.</p>
+<source>
+A = LOAD 'studenttab' USING org.apache.hadoop.zebra.pig.TableLoader('name, age');
+</source>
+ </section>
+ <!--end example: loading data-->
+
+ <!--example: map-side data-->
+ <section>
+ <title>Map-Side Group and Merge Join</title>
+
+<p>If the input data is globally sorted, map-side group or merge join can be used. Please, notice the âsortedâ argument passed to the loader. This lets the loader know that the data is expected to be globally sorted and that a single key must be given to the same map.</p>
+
+<p>Here is an example of the merge join. Note that the first argument to the loader is left empty to indicate that all columns are requested.</p>
+<source>
+A = LOAD'studentsortedtab' USING org.apache.hadoop.zebra.pig.TableLoader('', 'sorted');
+B = LOAD 'votersortedtab' USING org.apache.hadoop.zebra.pig.TableLoader('', 'sorted');
+G = JOIN A BY $0, B By $0 USING "merge";
+</source>
+
+<p>Here is an example of a map-side group. Note that multiple sorted files are passed to the loader and that the loader will perform sort preserving merge to make sure that the data is globally sorted.</p>
+<source>
+A = LOAD 'studentsortedtab, studentnullsortedtab' using org.apache.hadoop.zebra.pig.TableLoader('name, age, gpa, source_table', 'sorted');
+B = GROUP A BY $0 USING "collected";
+C = FOREACH B GENERATE group, MAX(a.$1);
+</source>
+ </section>
+<!--end example: map-side data-->
+
+
+
+ <!--example: storing data-->
+ <section>
+ <title>Storing Data</title>
+ <p>To store data in Zebra format use the TableStorer function. Since Zebra does not support anonymous columns, the relation that is stored must have a name (alias) assigned to every column in the relation.</p>
+<source>
+A = LOAD 'studentsortedtab, studentnullsortedtab' USING org.apache.hadoop.zebra.pig.TableLoader('name, age, gpa, source_table', 'sorted');
+B = GROUP A BY $0 USING "collected";
+C = FOREACH B GENERATE group, MAX(a.$1) AS max_val;
+STORE C INTO 'output' USING org.apache.hadoop.zebra.pig.TableStorer('');
+</source>
+</section>
+ <!--end example: storing data-->
+
+
+
+ <!--example: simple types-->
+ <section>
+ <title>Simple Types</title>
+ <source>
+-- The following line is required if the Zebra jar is not part of Hadoop Core
+
+REGISTER $LOCATION/zebra-$version.jar;
+
+-- Load an existing table (a projection string is not specified so all the columns will be loaded)
+
+A = LOAD '$PATH/tbl1' USING org.apache.hadoop.pig.zebra.pig.TableLoader();
+
+-- Store in table format with storage hint '[c1];[c2]' which mean two Column Groups with one column each (note that you do not specify a schema)
+
+STORE A INTO '$PATH/tbl2' USING org.apache.hadoop.zebra.pig.TableStorer('[c1] COMPRESS BY gz; SERIALIZE BY pig;[c2]');
+
+-- Load an existing table (two columns are projected)
+
+B = LOAD '$PATH/tbl3' USING org.apache.hadoop.zebra.pig.TableLoader('c1,c2');
+
+-- Load an existing table (one column is projected)
+
+C = LOAD '$PATH/tbl4' USING org.apache.hadoop.zebra.pig.TableLoader('c2');
+ </source>
+ </section>
+ <!--end example: simple types-->
+
+
+ <!--example: complex types-->
+ <section>
+ <title>Complex Types</title>
+ <source>
+-- The following line is required if the Zebra jar is not part of Hadoop Core
+
+register $LOCATION/zebra-$version.jar;
+
+-- STR_SCHEMA =
+-- "s1:bool, s2:int, s3:long, s4:float, s5:string, s6:bytes,
+-- r1:record(f1:int, f2:long), r2:record(r3:record(f3:float, f4)),
+-- m1:map(string),m2:map(map(int)),
+-- c:collection(f13:double, f14:float, f15:bytes)";
+
+-- STR_STORAGE =
+-- "[s1, s2]; [m1#{a}]; [r1.f1]; [s3, s4, r2.r3.f3];
+-- [s5, s6, m2#{x|y}]; [r1.f2, m1#{b}]; [r2.r3.f4, m2#{z}]";
+
+
+-- Load an existing table (two columns are projected)
+
+A = LOAD '$PATH/tbl1' USING org.apache.hadoop.zebra.pig.TableLoader('r1.f2, s1');
+
+-- Store in table format (all columns are stored in one column group)
+
+STORE A INTO '$PATH/tbl2' USING org.apache.hadoop.zebra.pig.TableStorer('');
+
+-- Load an existing table (two columns are projected)
+
+B = LOAD '$PATH/tbl3' USING org.apache.hadoop.zebra.pig.TableLoader('s1, r1');
+ </source>
+ </section>
+ <!--end example: complex types-->
+ </section>
+ <!-- END ZEBRA EXAMPLES-->
+
+ </body>
+ </document>
+
+