You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by pr...@apache.org on 2009/11/24 20:54:34 UTC
svn commit: r883836 [11/23] - in /hadoop/pig/branches/load-store-redesign:
./ contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/
contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/
contrib/zebra/ contrib/zebr...
Added: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestOrderPreserveSimple.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestOrderPreserveSimple.java?rev=883836&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestOrderPreserveSimple.java (added)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestOrderPreserveSimple.java Tue Nov 24 19:54:19 2009
@@ -0,0 +1,597 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.zebra.pig;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.ArrayList;
+import java.util.StringTokenizer;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.pig.TableStorer;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.executionengine.ExecJob;
+import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.test.MiniCluster;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+
+public class TestOrderPreserveSimple {
+
+ final static String STR_SCHEMA1 = "a:int,b:float,c:long,d:double,e:string,f:bytes,m1:map(string)";
+ final static String STR_STORAGE1 = "[a, b, c]; [d, e, f]; [m1#{a}]";
+
+ static int fileId = 0;
+
+ protected static ExecType execType = ExecType.MAPREDUCE;
+ private static MiniCluster cluster;
+ protected static PigServer pigServer;
+ protected static ExecJob pigJob;
+
+ private static Path pathTable1;
+ private static Path pathTable2;
+ private static Configuration conf;
+
+ private static Object[][] table1;
+ private static Object[][] table2;
+
+ private static Map<String, String> m1;
+ private static Map<String, String> m2;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ if (System.getProperty("hadoop.log.dir") == null) {
+ String base = new File(".").getPath(); // getAbsolutePath();
+ System.setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
+ }
+
+ if (execType == ExecType.MAPREDUCE) {
+ cluster = MiniCluster.buildCluster();
+ pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ } else {
+ pigServer = new PigServer(ExecType.LOCAL);
+ }
+
+ conf = new Configuration();
+ FileSystem fs = cluster.getFileSystem();
+ Path pathWorking = fs.getWorkingDirectory();
+
+ pathTable1 = new Path(pathWorking, "table1");
+ pathTable2 = new Path(pathWorking, "table2");
+
+ // Create table1 data
+ m1 = new HashMap<String, String>();
+ m1.put("a","m1-a");
+ m1.put("b","m1-b");
+
+ table1 = new Object[][]{
+ {5, -3.25f, 1001L, 51e+2, "Zebra", new DataByteArray("Zebra"), m1},
+ {-1, 3.25f, 1000L, 50e+2, "zebra", new DataByteArray("zebra"), m1},
+ {1001, 100.0f, 1003L, 50e+2, "Apple", new DataByteArray("Apple"), m1},
+ {1001, 101.0f, 1001L, 50e+2, "apple", new DataByteArray("apple"), m1},
+ {1001, 101.0f, 1001L, 50e+2, "apple", new DataByteArray("apple"), m1},
+ {1001, 50.0f, 1000L, 50e+2, "Pig", new DataByteArray("Pig"), m1},
+ {1001, 52.0f, 1001L, 50e+2, "pig", new DataByteArray("pig"), m1},
+ {1002, 28.0f, 1000L, 50e+2, "Hadoop", new DataByteArray("Hadoop"), m1},
+ {1000, 0.0f, 1002L, 52e+2, "hadoop", new DataByteArray("hadoop"), m1} };
+
+ // Create table1
+ createTable(pathTable1, STR_SCHEMA1, STR_STORAGE1, table1);
+
+ // Create table2 data
+ m2 = new HashMap<String, String>();
+ m2.put("a","m2-a");
+ m2.put("b","m2-b");
+
+ table2 = new Object[][] {
+ {15, 56.0f, 1004L, 50e+2, "green", new DataByteArray("green"), m2},
+ {-1, -99.0f, 1002L, 51e+2, "orange", new DataByteArray("orange"), m2},
+ {1001, 100.0f, 1003L, 55e+2, "white", new DataByteArray("white"), m2},
+ {1001, 102.0f, 1001L, 52e+2, "purple", new DataByteArray("purple"), m2},
+ {1001, 50.0f, 1008L, 52e+2, "gray", new DataByteArray("gray"), m2},
+ {1001, 53.0f, 1001L, 52e+2, "brown", new DataByteArray("brown"), m2},
+ {2000, 33.0f, 1006L, 52e+2, "beige", new DataByteArray("beige"), m2} };
+
+ // Create table2
+ createTable(pathTable2, STR_SCHEMA1, STR_STORAGE1, table2);
+ }
+
+ private static void createTable(Path path, String schemaString, String storageString, Object[][] tableData)
+ throws IOException {
+ //
+ // Create table from tableData array
+ //
+ BasicTable.Writer writer = new BasicTable.Writer(path, schemaString, storageString, conf);
+
+ Schema schema = writer.getSchema();
+ Tuple tuple = TypesUtils.createTuple(schema);
+ TableInserter inserter = writer.getInserter("ins", false);
+
+ for (int i = 0; i < tableData.length; ++i) {
+ TypesUtils.resetTuple(tuple);
+ for (int k = 0; k < tableData[i].length; ++k) {
+ tuple.set(k, tableData[i][k]);
+ System.out.println("DEBUG: setting tuple k=" + k + "value= " + tableData[i][k]);
+ }
+ inserter.insert(new BytesWritable(("key" + i).getBytes()), tuple);
+ }
+ inserter.close();
+ writer.close();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ pigServer.shutdown();
+ }
+
+ private Iterator<Tuple> orderPreserveUnion(String sortkey, String columns) throws IOException {
+ //
+ // Test sorted union with two tables that are different
+ //
+
+ // Load tables
+ String query1 = "table1 = LOAD '" + pathTable1.toString() + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query1);
+
+ String query2 = "table2 = LOAD '" + pathTable2.toString() + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query2);
+
+ // Sort tables
+ String orderby1 = "sort1 = ORDER table1 BY " + sortkey + " ;";
+ pigServer.registerQuery(orderby1);
+
+ String orderby2 = "sort2 = ORDER table2 BY " + sortkey + " ;";
+ pigServer.registerQuery(orderby2);
+
+ Path newPath = new Path(getCurrentMethodName());
+
+ // Store sorted tables
+ String pathSort1 = newPath.toString() + ++fileId; // increment fileId suffix
+ pigJob = pigServer.store("sort1", pathSort1, TableStorer.class.getCanonicalName() +
+ "('" + STR_STORAGE1 + "')");
+ Assert.assertNull(pigJob.getException());
+
+ String pathSort2 = newPath.toString() + ++fileId; // increment fileId suffix
+ pigJob = pigServer.store("sort2", pathSort2, TableStorer.class.getCanonicalName() +
+ "('" + STR_STORAGE1 + "')");
+ Assert.assertNull(pigJob.getException());
+
+ String queryLoad = "records1 = LOAD '"
+ + pathSort1 + ","
+ + pathSort2
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader('" + columns + "', 'sorted');";
+
+ System.out.println("queryLoad: " + queryLoad);
+
+ pigServer.registerQuery(queryLoad);
+
+ // Return iterator
+ Iterator<Tuple> it1 = pigServer.openIterator("records1");
+ return it1;
+ }
+
+ @Test
+ public void test_union_int() throws ExecException, IOException {
+ //
+ // Test sorted union with two tables with: int
+ //
+ orderPreserveUnion("a", "a,b,c,d,e,f,m1");
+
+ // Verify union table
+ ArrayList<ArrayList<Object>> resultTable = new ArrayList<ArrayList<Object>>();
+
+ addResultRow(resultTable, -1, 3.25f, 1000L, 50e+2, "zebra", new DataByteArray("zebra"), m1);
+ addResultRow(resultTable, -1, -99.0f, 1002L, 51e+2, "orange", new DataByteArray("orange"),m2);
+ addResultRow(resultTable, 5, -3.25f, 1001L, 51e+2, "Zebra", new DataByteArray("Zebra"), m1);
+ addResultRow(resultTable, 15, 56.0f, 1004L, 50e+2, "green", new DataByteArray("green"), m2);
+
+ addResultRow(resultTable, 1000, 0.0f, 1002L, 52e+2, "hadoop", new DataByteArray("hadoop"),m1);
+ addResultRow(resultTable, 1001, 100.0f, 1003L, 55e+2, "white", new DataByteArray("white"), m2);
+ addResultRow(resultTable, 1001, 100.0f, 1003L, 50e+2, "Apple", new DataByteArray("Apple"), m1);
+ addResultRow(resultTable, 1001, 102.0f, 1001L, 52e+2, "purple", new DataByteArray("purple"),m2);
+
+ addResultRow(resultTable, 1001, 101.0f, 1001L, 50e+2, "apple", new DataByteArray("apple"), m1);
+ addResultRow(resultTable, 1001, 50.0f, 1008L, 52e+2, "gray", new DataByteArray("gray"), m2);
+ addResultRow(resultTable, 1001, 101.0f, 1001L, 50e+2, "apple", new DataByteArray("apple"), m1);
+ addResultRow(resultTable, 1001, 53.0f, 1001L, 52e+2, "brown", new DataByteArray("brown"), m2);
+
+ addResultRow(resultTable, 1001, 50.0f, 1000L, 50e+2, "Pig", new DataByteArray("Pig"), m1);
+ addResultRow(resultTable, 1001, 52.0f, 1001L, 50e+2, "pig", new DataByteArray("pig"), m1);
+ addResultRow(resultTable, 1002, 28.0f, 1000L, 50e+2, "Hadoop", new DataByteArray("Hadoop"),m1);
+ addResultRow(resultTable, 2000, 33.0f, 1006L, 52e+2, "beige", new DataByteArray("beige"), m2);
+
+ // Verify union table
+ Iterator<Tuple> it = pigServer.openIterator("records1");
+ int numbRows = verifyTable(resultTable, 0, it);
+
+ Assert.assertEquals(numbRows, table1.length + table2.length);
+ }
+
+ @Test
+ public void test_union_int_source() throws ExecException, IOException {
+ //
+ // Test sorted union with two tables with: int
+ //
+ orderPreserveUnion("a", "source_table,a,b,c,d,e,f,m1");
+
+ // Verify union table
+ ArrayList<ArrayList<Object>> resultTable = new ArrayList<ArrayList<Object>>();
+
+ addResultRow(resultTable, 0, -1, 3.25f, 1000L, 50e+2, "zebra", new DataByteArray("zebra"), m1);
+ addResultRow(resultTable, 1, -1, -99.0f, 1002L, 51e+2, "orange", new DataByteArray("orange"),m2);
+ addResultRow(resultTable, 0, 5, -3.25f, 1001L, 51e+2, "Zebra", new DataByteArray("Zebra"), m1);
+ addResultRow(resultTable, 1, 15, 56.0f, 1004L, 50e+2, "green", new DataByteArray("green"), m2);
+
+ addResultRow(resultTable, 0, 1000, 0.0f, 1002L, 52e+2, "hadoop", new DataByteArray("hadoop"),m1);
+ addResultRow(resultTable, 1, 1001, 100.0f, 1003L, 55e+2, "white", new DataByteArray("white"), m2);
+ addResultRow(resultTable, 0, 1001, 100.0f, 1003L, 50e+2, "Apple", new DataByteArray("Apple"), m1);
+ addResultRow(resultTable, 1, 1001, 102.0f, 1001L, 52e+2, "purple", new DataByteArray("purple"),m2);
+
+ addResultRow(resultTable, 0, 1001, 101.0f, 1001L, 50e+2, "apple", new DataByteArray("apple"), m1);
+ addResultRow(resultTable, 1, 1001, 50.0f, 1008L, 52e+2, "gray", new DataByteArray("gray"), m2);
+ addResultRow(resultTable, 0, 1001, 101.0f, 1001L, 50e+2, "apple", new DataByteArray("apple"), m1);
+ addResultRow(resultTable, 1, 1001, 53.0f, 1001L, 52e+2, "brown", new DataByteArray("brown"), m2);
+
+ addResultRow(resultTable, 0, 1001, 50.0f, 1000L, 50e+2, "Pig", new DataByteArray("Pig"), m1);
+ addResultRow(resultTable, 0, 1001, 52.0f, 1001L, 50e+2, "pig", new DataByteArray("pig"), m1);
+ addResultRow(resultTable, 0, 1002, 28.0f, 1000L, 50e+2, "Hadoop", new DataByteArray("Hadoop"),m1);
+ addResultRow(resultTable, 1, 2000, 33.0f, 1006L, 52e+2, "beige", new DataByteArray("beige"), m2);
+
+ // Verify union table
+ Iterator<Tuple> it = pigServer.openIterator("records1");
+ int numbRows = verifyTable(resultTable, 1, it);
+
+ Assert.assertEquals(numbRows, table1.length + table2.length);
+ }
+
+ @Test
+ public void test_union_float_source() throws ExecException, IOException {
+ //
+ // Test sorted union with two tables with: float
+ //
+ orderPreserveUnion("b", "source_table,b,a,c,d,e,f,m1");
+
+ // Verify union table
+ ArrayList<ArrayList<Object>> resultTable = new ArrayList<ArrayList<Object>>();
+
+ addResultRow(resultTable, 1, -99.0f, -1, 1002L, 51e+2, "orange", new DataByteArray("orange"),m2);
+ addResultRow(resultTable, 0, -3.25f, 5, 1001L, 51e+2, "Zebra", new DataByteArray("Zebra"), m1);
+ addResultRow(resultTable, 0, 0.0f, 1000, 1002L, 52e+2, "hadoop", new DataByteArray("hadoop"),m1);
+ addResultRow(resultTable, 0, 3.25f, -1, 1000L, 50e+2, "zebra", new DataByteArray("zebra"), m1);
+
+ addResultRow(resultTable, 0, 28.0f, 1002, 1000L, 50e+2, "Hadoop", new DataByteArray("Hadoop"),m1);
+ addResultRow(resultTable, 1, 33.0f, 2000, 1006L, 52e+2, "beige", new DataByteArray("beige"), m2);
+ addResultRow(resultTable, 0, 50.0f, 1001, 1000L, 50e+2, "Pig", new DataByteArray("Pig"), m1);
+ addResultRow(resultTable, 1, 50.0f, 1001, 1008L, 52e+2, "gray", new DataByteArray("gray"), m2);
+
+ addResultRow(resultTable, 0, 52.0f, 1001, 1001L, 50e+2, "pig", new DataByteArray("pig"), m1);
+ addResultRow(resultTable, 1, 53.0f, 1001, 1001L, 52e+2, "brown", new DataByteArray("brown"), m2);
+ addResultRow(resultTable, 1, 56.0f, 15, 1004L, 50e+2, "green", new DataByteArray("green"), m2);
+ addResultRow(resultTable, 0, 100.0f, 1001, 1003L, 50e+2, "Apple", new DataByteArray("Apple"), m1);
+
+ addResultRow(resultTable, 1, 100.0f, 1001, 1003L, 55e+2, "white", new DataByteArray("white"), m2);
+ addResultRow(resultTable, 0, 101.0f, 1001, 1001L, 50e+2, "apple", new DataByteArray("apple"), m1);
+ addResultRow(resultTable, 0, 101.0f, 1001, 1001L, 50e+2, "apple", new DataByteArray("apple"), m1);
+ addResultRow(resultTable, 1, 102.0f, 1001, 1001L, 52e+2, "purple", new DataByteArray("purple"),m2);
+
+ // Verify union table
+ Iterator<Tuple> it = pigServer.openIterator("records1");
+ int numbRows = verifyTable(resultTable, 1, it);
+
+ Assert.assertEquals(numbRows, table1.length + table2.length);
+ }
+
+ @Test
+ public void test_union_long_source() throws ExecException, IOException {
+ //
+ // Test sorted union with two tables with: long
+ //
+ orderPreserveUnion("c", "source_table,c,a,b,d,e,f,m1");
+
+ // Verify union table
+ ArrayList<ArrayList<Object>> resultTable = new ArrayList<ArrayList<Object>>();
+
+ addResultRow(resultTable, 0, 1000L, -1, 3.25f, 50e+2, "zebra", new DataByteArray("zebra"), m1);
+ addResultRow(resultTable, 0, 1000L, 1001, 50.0f, 50e+2, "Pig", new DataByteArray("Pig"), m1);
+ addResultRow(resultTable, 0, 1000L, 1002, 28.0f, 50e+2, "Hadoop", new DataByteArray("Hadoop"),m1);
+ addResultRow(resultTable, 1, 1001L, 1001, 102.0f, 52e+2, "purple", new DataByteArray("purple"),m2);
+
+ addResultRow(resultTable, 0, 1001L, 5, -3.25f, 51e+2, "Zebra", new DataByteArray("Zebra"), m1);
+ addResultRow(resultTable, 1, 1001L, 1001, 53.0f, 52e+2, "brown", new DataByteArray("brown"), m2);
+ addResultRow(resultTable, 0, 1001L, 1001, 101.0f, 50e+2, "apple", new DataByteArray("apple"), m1);
+ addResultRow(resultTable, 0, 1001L, 1001, 101.0f, 50e+2, "apple", new DataByteArray("apple"), m1);
+
+ addResultRow(resultTable, 0, 1001L, 1001, 52.0f, 50e+2, "pig", new DataByteArray("pig"), m1);
+ addResultRow(resultTable, 1, 1002L, -1, -99.0f, 51e+2, "orange", new DataByteArray("orange"),m2);
+ addResultRow(resultTable, 0, 1002L, 1000, 0.0f, 52e+2, "hadoop", new DataByteArray("hadoop"),m1);
+ addResultRow(resultTable, 1, 1003L, 1001, 100.0f, 55e+2, "white", new DataByteArray("white"), m2);
+
+ addResultRow(resultTable, 0, 1003L, 1001, 100.0f, 50e+2, "Apple", new DataByteArray("Apple"), m1);
+ addResultRow(resultTable, 1, 1004L, 15, 56.0f, 50e+2, "green", new DataByteArray("green"), m2);
+ addResultRow(resultTable, 1, 1006L, 2000, 33.0f, 52e+2, "beige", new DataByteArray("beige"), m2);
+ addResultRow(resultTable, 1, 1008L, 1001, 50.0f, 52e+2, "gray", new DataByteArray("gray"), m2);
+
+ // Verify union table
+ Iterator<Tuple> it = pigServer.openIterator("records1");
+ int numbRows = verifyTable(resultTable, 1, it);
+
+ Assert.assertEquals(numbRows, table1.length + table2.length);
+ }
+
+ @Test
+ public void test_union_double_source() throws ExecException, IOException {
+ //
+ // Test sorted union with two tables with: double
+ //
+ orderPreserveUnion("d", "source_table,d,a,b,c,e,f,m1");
+
+ // Verify union table
+ ArrayList<ArrayList<Object>> resultTable = new ArrayList<ArrayList<Object>>();
+
+ addResultRow(resultTable, 0,50e+2, -1, 3.25f, 1000L, "zebra", new DataByteArray("zebra"), m1);
+ addResultRow(resultTable, 1,50e+2, 15, 56.0f, 1004L, "green", new DataByteArray("green"), m2);
+ addResultRow(resultTable, 0,50e+2, 1001, 100.0f, 1003L, "Apple", new DataByteArray("Apple"), m1);
+ addResultRow(resultTable, 0,50e+2, 1001, 101.0f, 1001L, "apple", new DataByteArray("apple"), m1);
+
+ addResultRow(resultTable, 0,50e+2, 1001, 101.0f, 1001L, "apple", new DataByteArray("apple"), m1);
+ addResultRow(resultTable, 0,50e+2, 1001, 50.0f, 1000L, "Pig", new DataByteArray("Pig"), m1);
+ addResultRow(resultTable, 0,50e+2, 1001, 52.0f, 1001L, "pig", new DataByteArray("pig"), m1);
+ addResultRow(resultTable, 0,50e+2, 1002, 28.0f, 1000L, "Hadoop", new DataByteArray("Hadoop"),m1);
+
+ addResultRow(resultTable, 1,51e+2, -1, -99.0f, 1002L, "orange", new DataByteArray("orange"),m2);
+ addResultRow(resultTable, 0,51e+2, 5, -3.25f, 1001L, "Zebra", new DataByteArray("Zebra"), m1);
+ addResultRow(resultTable, 1,52e+2, 1001, 102.0f, 1001L, "purple", new DataByteArray("purple"),m2);
+ addResultRow(resultTable, 0,52e+2, 1000, 0.0f, 1002L, "hadoop", new DataByteArray("hadoop"),m1);
+
+ addResultRow(resultTable, 1,52e+2, 1001, 50.0f, 1008L, "gray", new DataByteArray("gray"), m2);
+ addResultRow(resultTable, 1,52e+2, 1001, 53.0f, 1001L, "brown", new DataByteArray("brown"), m2);
+ addResultRow(resultTable, 1,52e+2, 2000, 33.0f, 1006L, "beige", new DataByteArray("beige"), m2);
+ addResultRow(resultTable, 1,55e+2, 1001, 100.0f, 1003L, "white", new DataByteArray("white"), m2);
+
+ // Verify union table
+ Iterator<Tuple> it = pigServer.openIterator("records1");
+ int numbRows = verifyTable(resultTable, 1, it);
+
+ Assert.assertEquals(numbRows, table1.length + table2.length);
+ }
+
+ @Test
+ public void test_union_string_source() throws ExecException, IOException {
+ //
+ // Test sorted union with two tables with: string
+ //
+ orderPreserveUnion("e", "source_table,e,a,b,c,d,f,m1");
+
+ // Verify union table
+ ArrayList<ArrayList<Object>> resultTable = new ArrayList<ArrayList<Object>>();
+
+ addResultRow(resultTable, 0,"Apple", 1001, 100.0f, 1003L, 50e+2, new DataByteArray("Apple"), m1);
+ addResultRow(resultTable, 0,"Hadoop", 1002, 28.0f, 1000L, 50e+2, new DataByteArray("Hadoop"),m1);
+ addResultRow(resultTable, 0,"Pig", 1001, 50.0f, 1000L, 50e+2, new DataByteArray("Pig"), m1);
+ addResultRow(resultTable, 0,"Zebra", 5, -3.25f, 1001L, 51e+2, new DataByteArray("Zebra"), m1);
+
+ addResultRow(resultTable, 0,"apple", 1001, 101.0f, 1001L, 50e+2, new DataByteArray("apple"), m1);
+ addResultRow(resultTable, 0,"apple", 1001, 101.0f, 1001L, 50e+2, new DataByteArray("apple"), m1);
+ addResultRow(resultTable, 1,"beige", 2000, 33.0f, 1006L, 52e+2, new DataByteArray("beige"), m2);
+ addResultRow(resultTable, 1,"brown", 1001, 53.0f, 1001L, 52e+2, new DataByteArray("brown"), m2);
+
+ addResultRow(resultTable, 1,"gray", 1001, 50.0f, 1008L, 52e+2, new DataByteArray("gray"), m2);
+ addResultRow(resultTable, 1,"green", 15, 56.0f, 1004L, 50e+2, new DataByteArray("green"), m2);
+ addResultRow(resultTable, 0,"hadoop", 1000, 0.0f, 1002L, 52e+2, new DataByteArray("hadoop"),m1);
+ addResultRow(resultTable, 1,"orange", -1, -99.0f, 1002L, 51e+2, new DataByteArray("orange"),m2);
+
+ addResultRow(resultTable, 0,"pig", 1001, 52.0f, 1001L, 50e+2, new DataByteArray("pig"), m1);
+ addResultRow(resultTable, 1,"purple", 1001, 102.0f, 1001L, 52e+2, new DataByteArray("purple"),m2);
+ addResultRow(resultTable, 1,"white", 1001, 100.0f, 1003L, 55e+2, new DataByteArray("white"), m2);
+ addResultRow(resultTable, 0,"zebra", -1, 3.25f, 1000L, 50e+2, new DataByteArray("zebra"), m1);
+
+ // Verify union table
+ Iterator<Tuple> it = pigServer.openIterator("records1");
+ int numbRows = verifyTable(resultTable, 1, it);
+
+ Assert.assertEquals(numbRows, table1.length + table2.length);
+ }
+
+ @Test
+ public void test_union_bytes_source() throws ExecException, IOException {
+ //
+ // Test sorted union with two tables with: bytes
+ //
+ orderPreserveUnion("f", "source_table,f,a,b,c,d,e,m1");
+
+ // Verify union table
+ ArrayList<ArrayList<Object>> resultTable = new ArrayList<ArrayList<Object>>();
+
+ addResultRow(resultTable, 0,new DataByteArray("Apple"), 1001, 100.0f, 1003L, 50e+2, "Apple", m1);
+ addResultRow(resultTable, 0,new DataByteArray("Hadoop"),1002, 28.0f, 1000L, 50e+2, "Hadoop", m1);
+ addResultRow(resultTable, 0,new DataByteArray("Pig"), 1001, 50.0f, 1000L, 50e+2, "Pig", m1);
+ addResultRow(resultTable, 0,new DataByteArray("Zebra"), 5, -3.25f, 1001L, 51e+2, "Zebra", m1);
+
+ addResultRow(resultTable, 0,new DataByteArray("apple"), 1001, 101.0f, 1001L, 50e+2, "apple", m1);
+ addResultRow(resultTable, 0,new DataByteArray("apple"), 1001, 101.0f, 1001L, 50e+2, "apple", m1);
+ addResultRow(resultTable, 1,new DataByteArray("beige"), 2000, 33.0f, 1006L, 52e+2, "beige", m2);
+ addResultRow(resultTable, 1,new DataByteArray("brown"), 1001, 53.0f, 1001L, 52e+2, "brown", m2);
+
+ addResultRow(resultTable, 1,new DataByteArray("gray"), 1001, 50.0f, 1008L, 52e+2, "gray", m2);
+ addResultRow(resultTable, 1,new DataByteArray("green"), 15, 56.0f, 1004L, 50e+2, "green", m2);
+ addResultRow(resultTable, 0,new DataByteArray("hadoop"),1000, 0.0f, 1002L, 52e+2, "hadoop", m1);
+ addResultRow(resultTable, 1,new DataByteArray("orange"),-1, -99.0f, 1002L, 51e+2, "orange", m2);
+
+ addResultRow(resultTable, 0,new DataByteArray("pig"), 1001, 52.0f, 1001L, 50e+2, "pig", m1);
+ addResultRow(resultTable, 1,new DataByteArray("purple"),1001, 102.0f, 1001L, 52e+2, "purple", m2);
+ addResultRow(resultTable, 1,new DataByteArray("white"), 1001, 100.0f, 1003L, 55e+2, "white", m2);
+ addResultRow(resultTable, 0,new DataByteArray("zebra"), -1, 3.25f, 1000L, 50e+2, "zebra", m1);
+
+ // Verify union table
+ Iterator<Tuple> it = pigServer.openIterator("records1");
+ int numbRows = verifyTable(resultTable, 1, it);
+
+ Assert.assertEquals(numbRows, table1.length + table2.length);
+ }
+
+ /**
+ * Verify union output table with expected results
+ *
+ */
+ private int verifyTable(ArrayList<ArrayList<Object>> resultTable, int keyColumn, Iterator<Tuple> it) throws IOException {
+ int numbRows = 0;
+ int index = 0;
+ Object value = resultTable.get(index).get(keyColumn); // get value of primary key
+
+ while (it.hasNext()) {
+ Tuple rowValues = it.next();
+
+ // If last primary sort key does match then search for next matching key
+ if (! compareObj(value, rowValues.get(keyColumn))) {
+ int subIndex = index + 1;
+ while (subIndex < resultTable.size()) {
+ if ( ! compareObj(value, resultTable.get(subIndex).get(keyColumn)) ) { // found new key
+ index = subIndex;
+ value = resultTable.get(index).get(keyColumn);
+ break;
+ }
+ ++subIndex;
+ }
+ Assert.assertEquals("Table comparison error for row : " + numbRows + " - no key found for : "
+ + rowValues.get(keyColumn), value, rowValues.get(keyColumn));
+ }
+ // Search for matching row with this primary key
+ int subIndex = index;
+
+ while (subIndex < resultTable.size()) {
+ // Compare row
+ ArrayList<Object> resultRow = resultTable.get(subIndex);
+ if ( compareRow(rowValues, resultRow) )
+ break; // found matching row
+ ++subIndex;
+ Assert.assertEquals("Table comparison error for row : " + numbRows + " - no matching row found for : "
+ + rowValues.get(keyColumn), value, resultTable.get(subIndex).get(keyColumn));
+ }
+ ++numbRows;
+ }
+ Assert.assertEquals(resultTable.size(), numbRows); // verify expected row count
+ return numbRows;
+ }
+
+ /**
+ * Compare table rows
+ *
+ */
+ private boolean compareRow(Tuple rowValues, ArrayList<Object> resultRow) throws IOException {
+ boolean result = true;
+ Assert.assertEquals(resultRow.size(), rowValues.size());
+ for (int i = 0; i < rowValues.size(); ++i) {
+ if (! compareObj(rowValues.get(i), resultRow.get(i)) ) {
+ result = false;
+ break;
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Compare table values
+ *
+ */
+ private boolean compareObj(Object object1, Object object2) {
+ if (object1 == null) {
+ if (object2 == null)
+ return true;
+ else
+ return false;
+ } else if (object1.equals(object2))
+ return true;
+ else
+ return false;
+ }
+
+ /**
+ *Add a row to expected results table
+ *
+ */
+ private void addResultRow(ArrayList<ArrayList<Object>> resultTable, Object ... values) {
+ ArrayList<Object> resultRow = new ArrayList<Object>();
+
+ for (int i = 0; i < values.length; i++) {
+ resultRow.add(values[i]);
+ }
+ resultTable.add(resultRow);
+ }
+
+ /**
+ * Print Pig Table (for debugging)
+ *
+ */
+ private int printTable(String tablename) throws IOException {
+ Iterator<Tuple> it1 = pigServer.openIterator(tablename);
+ int numbRows = 0;
+ while (it1.hasNext()) {
+ Tuple RowValue1 = it1.next();
+ ++numbRows;
+ System.out.println();
+ for (int i = 0; i < RowValue1.size(); ++i)
+ System.out.println("DEBUG: " + tablename + " RowValue.get(" + i + ") = " + RowValue1.get(i));
+ }
+ System.out.println("\nRow count : " + numbRows);
+ return numbRows;
+ }
+
+ /**
+ * Return the name of the routine that called getCurrentMethodName
+ *
+ */
+ public String getCurrentMethodName() {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ (new Throwable()).printStackTrace(pw);
+ pw.flush();
+ String stackTrace = baos.toString();
+ pw.close();
+
+ StringTokenizer tok = new StringTokenizer(stackTrace, "\n");
+ tok.nextToken(); // 'java.lang.Throwable'
+ tok.nextToken(); // 'at ...getCurrentMethodName'
+ String l = tok.nextToken(); // 'at ...<caller to getCurrentRoutine>'
+ // Parse line 3
+ tok = new StringTokenizer(l.trim(), " <(");
+ String t = tok.nextToken(); // 'at'
+ t = tok.nextToken(); // '...<caller to getCurrentRoutine>'
+ return t;
+ }
+
+}
Added: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestOrderPreserveUnion.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestOrderPreserveUnion.java?rev=883836&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestOrderPreserveUnion.java (added)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestOrderPreserveUnion.java Tue Nov 24 19:54:19 2009
@@ -0,0 +1,227 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.zebra.pig;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.StringTokenizer;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.pig.TableStorer;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.test.MiniCluster;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Note:
+ *
+ * Make sure you add the build/pig-0.1.0-dev-core.jar to the Classpath of the
+ * app/debug configuration, when run this from inside the Eclipse.
+ *
+ */
+public class TestOrderPreserveUnion {
+ protected static ExecType execType = ExecType.MAPREDUCE;
+ private static MiniCluster cluster;
+ protected static PigServer pigServer;
+ private static Path pathTable;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ if (System.getProperty("hadoop.log.dir") == null) {
+ String base = new File(".").getPath(); // getAbsolutePath();
+ System
+ .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
+ }
+
+ if (execType == ExecType.MAPREDUCE) {
+ cluster = MiniCluster.buildCluster();
+ pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ } else {
+ pigServer = new PigServer(ExecType.LOCAL);
+ }
+
+ Configuration conf = new Configuration();
+ FileSystem fs = cluster.getFileSystem();
+ Path pathWorking = fs.getWorkingDirectory();
+ pathTable = new Path(pathWorking, "TestTableStorer");
+ System.out.println("pathTable =" + pathTable);
+ BasicTable.Writer writer = new BasicTable.Writer(pathTable,
+ "SF_a:string,SF_b:string,SF_c,SF_d,SF_e,SF_f,SF_g",
+ "[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]", conf);
+ Schema schema = writer.getSchema();
+ Tuple tuple = TypesUtils.createTuple(schema);
+
+ final int numsBatch = 10;
+ final int numsInserters = 1;
+ TableInserter[] inserters = new TableInserter[numsInserters];
+ for (int i = 0; i < numsInserters; i++) {
+ inserters[i] = writer.getInserter("ins" + i, false);
+ }
+
+ for (int b = 0; b < numsBatch; b++) {
+ for (int i = 0; i < numsInserters; i++) {
+ TypesUtils.resetTuple(tuple);
+ for (int k = 0; k < tuple.size(); ++k) {
+ try {
+ tuple.set(k, (9-b) + "_" + i + "" + k);
+ } catch (ExecException e) {
+ e.printStackTrace();
+ }
+ }
+ inserters[i].insert(new BytesWritable(("key" + i).getBytes()), tuple);
+ }
+ }
+ for (int i = 0; i < numsInserters; i++) {
+ inserters[i].close();
+ }
+ writer.close();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ pigServer.shutdown();
+ }
+
+ /**
+ * Return the name of the routine that called getCurrentMethodName
+ *
+ */
+ public String getCurrentMethodName() {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ (new Throwable()).printStackTrace(pw);
+ pw.flush();
+ String stackTrace = baos.toString();
+ pw.close();
+
+ StringTokenizer tok = new StringTokenizer(stackTrace, "\n");
+ tok.nextToken(); // 'java.lang.Throwable'
+ tok.nextToken(); // 'at ...getCurrentMethodName'
+ String l = tok.nextToken(); // 'at ...<caller to getCurrentRoutine>'
+ // Parse line 3
+ tok = new StringTokenizer(l.trim(), " <(");
+ String t = tok.nextToken(); // 'at'
+ t = tok.nextToken(); // '...<caller to getCurrentRoutine>'
+ return t;
+ }
+
+ @Test
+ public void testStorer() throws ExecException, IOException {
+ /*
+ * Use pig LOAD to load testing data for store
+ */
+ String query = "records = LOAD '" + pathTable.toString()
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query);
+
+ /*
+ Iterator<Tuple> it2 = pigServer.openIterator("records");
+ int row0 = 0;
+ Tuple RowValue2 = null;
+ while (it2.hasNext()) {
+ // Last row value
+ RowValue2 = it2.next();
+ row0++;
+ if (row0 == 10) {
+ Assert.assertEquals("0_01", RowValue2.get(1));
+ Assert.assertEquals("0_00", RowValue2.get(0));
+ }
+ }
+ Assert.assertEquals(10, row0);
+ */
+
+ String orderby = "srecs = ORDER records BY SF_a;";
+ pigServer.registerQuery(orderby);
+
+ /*
+ * Use pig STORE to store testing data BasicTable.Writer writer = new
+ * BasicTable.Writer(pathTable, "SF_a,SF_b,SF_c,SF_d,SF_e,SF_f,SF_g",
+ * "[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]", false, conf);
+ */
+ Path newPath = new Path(getCurrentMethodName());
+
+ /*
+ * Table1 creation
+ */
+ pigServer
+ .store(
+ "srecs",
+ newPath.toString()+"1",
+ TableStorer.class.getCanonicalName()
+ + "('[SF_a, SF_b, SF_c]; [SF_e]')");
+ pigServer
+ .store(
+ "srecs",
+ newPath.toString()+"2",
+ TableStorer.class.getCanonicalName()
+ + "('[SF_a, SF_b, SF_c]; [SF_e]')");
+
+
+ String query4 = "records2 = LOAD '"
+ + newPath.toString() + "1,"
+ + newPath.toString() + "2"
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader('SF_a, source_table, SF_b', 'sorted');";
+ pigServer.registerQuery(query4);
+
+ // check JOIN content
+ Iterator<Tuple> it3 = pigServer.openIterator("records2");
+ int row = 0, index, prevtindex = -1, tindex;
+ Tuple RowValue3 = null;
+ while (it3.hasNext()) {
+ // Last row value
+ RowValue3 = it3.next();
+ Assert.assertEquals(3, RowValue3.size());
+ row++;
+ index = (row-1)/2;
+ if (prevtindex != -1)
+ {
+ // the other index is expected
+ tindex = 1 - prevtindex;
+ prevtindex = -1;
+ } else {
+ tindex = (Integer) RowValue3.get(1);
+ prevtindex = tindex;
+ }
+ Assert.assertEquals(index+"_00", RowValue3.get(0));
+ Assert.assertEquals(index+"_01", RowValue3.get(2));
+ Assert.assertEquals(tindex, RowValue3.get(1));
+ }
+ Assert.assertEquals(20, row);
+ }
+}
Added: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestOrderPreserveVariableTable.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestOrderPreserveVariableTable.java?rev=883836&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestOrderPreserveVariableTable.java (added)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestOrderPreserveVariableTable.java Tue Nov 24 19:54:19 2009
@@ -0,0 +1,828 @@
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.zebra.pig;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.ArrayList;
+import java.util.StringTokenizer;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.pig.TableStorer;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.executionengine.ExecJob;
+import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.test.MiniCluster;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+
+public class TestOrderPreserveVariableTable {
+
+ final static String TABLE1_SCHEMA = "a:int,b:float,c:long,d:double,e:string,f:bytes,m1:map(string)";
+ final static String TABLE1_STORAGE = "[a, b, c]; [d, e, f]; [m1#{a}]";
+
+ final static String TABLE2_SCHEMA = "a:int,b:float,c:long,d:double,e:string,f:bytes,m1:map(string)";
+ final static String TABLE2_STORAGE = "[a, b, c]; [d, e, f]; [m1#{a}]";
+
+ final static String TABLE3_SCHEMA = "b:float,d:double,c:long,e:string,f:bytes,int1:int";
+ final static String TABLE3_STORAGE = "[b,d]; [c]; [e]; [f,int1]";
+
+ final static String TABLE4_SCHEMA = "e:string,str3:string,str4:string";
+ final static String TABLE4_STORAGE = "[e,str3,str4]";
+
+ static int fileId = 0;
+ static int sortId = 0;
+
+ protected static ExecType execType = ExecType.MAPREDUCE;
+ private static MiniCluster cluster;
+ protected static PigServer pigServer;
+ protected static ExecJob pigJob;
+
+ private static Path pathTable1;
+ private static Path pathTable2;
+ private static Path pathTable3;
+ private static Path pathTable4;
+ private static HashMap<String, String> tableStorage;
+
+ private static Configuration conf;
+
+ private static Object[][] table1;
+ private static Object[][] table2;
+ private static Object[][] table3;
+ private static Object[][] table4;
+
+ private static Map<String, String> m1;
+ private static Map<String, String> m2;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ if (System.getProperty("hadoop.log.dir") == null) {
+ String base = new File(".").getPath(); // getAbsolutePath();
+ System.setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
+ }
+
+ if (execType == ExecType.MAPREDUCE) {
+ cluster = MiniCluster.buildCluster();
+ pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ } else {
+ pigServer = new PigServer(ExecType.LOCAL);
+ }
+
+ conf = new Configuration();
+ FileSystem fs = cluster.getFileSystem();
+ Path pathWorking = fs.getWorkingDirectory();
+
+ pathTable1 = new Path(pathWorking, "table1");
+ pathTable2 = new Path(pathWorking, "table2");
+ pathTable3 = new Path(pathWorking, "table3");
+ pathTable4 = new Path(pathWorking, "table4");
+
+ // Create table1 data
+ m1 = new HashMap<String, String>();
+ m1.put("a","m1-a");
+ m1.put("b","m1-b");
+
+ table1 = new Object[][]{
+ {5, -3.25f, 1001L, 51e+2, "Zebra", new DataByteArray("Zebra"), m1},
+ {-1, 3.25f, 1000L, 50e+2, "zebra", new DataByteArray("zebra"), m1},
+ {1001, 100.0f, 1003L, 50e+2, "Apple", new DataByteArray("Apple"), m1},
+ {1001, 101.0f, 1001L, 50e+2, "apple", new DataByteArray("apple"), m1},
+ {1001, 50.0f, 1000L, 50e+2, "Pig", new DataByteArray("Pig"), m1},
+ {1001, 52.0f, 1001L, 50e+2, "pig", new DataByteArray("pig"), m1},
+ {1002, 28.0f, 1000L, 50e+2, "Hadoop", new DataByteArray("Hadoop"), m1},
+ {1000, 0.0f, 1002L, 52e+2, "hadoop", new DataByteArray("hadoop"), m1} };
+
+ // Create table1
+ createTable(pathTable1, TABLE1_SCHEMA, TABLE1_STORAGE, table1);
+
+ // Create table2 data
+ m2 = new HashMap<String, String>();
+ m2.put("a","m2-a");
+ m2.put("b","m2-b");
+
+ table2 = new Object[][] {
+ {15, 56.0f, 1004L, 50e+2, "green", new DataByteArray("green"), m2},
+ {-1, -99.0f, 1002L, 51e+2, "orange", new DataByteArray("orange"), m2},
+ {1001, 100.0f, 1003L, 55e+2, "white", new DataByteArray("white"), m2},
+ {1001, 102.0f, 1001L, 52e+2, "purple", new DataByteArray("purple"), m2},
+ {1001, 50.0f, 1008L, 52e+2, "gray", new DataByteArray("gray"), m2},
+ {1001, 53.0f, 1001L, 52e+2, "brown", new DataByteArray("brown"), m2},
+ {2000, 33.0f, 1006L, 52e+2, "beige", new DataByteArray("beige"), m2} };
+
+ // Create table2
+ createTable(pathTable2, TABLE2_SCHEMA, TABLE2_STORAGE, table2);
+
+ // Create table3 data
+ table3 = new Object[][] {
+ {3.25f, 51e+2, 1001L, "string1", new DataByteArray("green"), 100}, };
+
+ // Create table3
+ createTable(pathTable3, TABLE3_SCHEMA, TABLE3_STORAGE, table3);
+
+ // Create table4 data
+ table4 = new Object[][] {
+ {"a 8", "Cupertino", "California"},
+ {"a 7", "San Jose", "California"},
+ {"a 6", "Santa Cruz", "California"},
+ {"a 5", "Las Vegas", "Nevada"},
+ {"a 4", "New York", "New York"},
+ {"a 3", "Phoenix", "Arizona"},
+ {"a 2", "Dallas", "Texas"},
+ {"a 1", "Reno", "Nevada"} };
+
+ // Create table4
+ createTable(pathTable4, TABLE4_SCHEMA, TABLE4_STORAGE, table4);
+
+ // Load tables
+ String query1 = "table1 = LOAD '" + pathTable1.toString() + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query1);
+ String query2 = "table2 = LOAD '" + pathTable2.toString() + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query2);
+ String query3 = "table3 = LOAD '" + pathTable3.toString() + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query3);
+ String query4 = "table4 = LOAD '" + pathTable4.toString() + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query4);
+
+ // Map for table storage
+ tableStorage = new HashMap<String, String>();
+ tableStorage.put("table1", TABLE1_STORAGE);
+ tableStorage.put("table2", TABLE2_STORAGE);
+ tableStorage.put("table3", TABLE3_STORAGE);
+ tableStorage.put("table4", TABLE4_STORAGE);
+ }
+
+ private static void createTable(Path path, String schemaString, String storageString, Object[][] tableData)
+ throws IOException {
+ //
+ // Create table from tableData array
+ //
+ BasicTable.Writer writer = new BasicTable.Writer(path, schemaString, storageString, conf);
+
+ Schema schema = writer.getSchema();
+ Tuple tuple = TypesUtils.createTuple(schema);
+ TableInserter inserter = writer.getInserter("ins", false);
+
+ for (int i = 0; i < tableData.length; ++i) {
+ TypesUtils.resetTuple(tuple);
+ for (int k = 0; k < tableData[i].length; ++k) {
+ tuple.set(k, tableData[i][k]);
+ System.out.println("DEBUG: setting tuple k=" + k + "value= " + tableData[i][k]);
+ }
+ inserter.insert(new BytesWritable(("key" + i).getBytes()), tuple);
+ }
+ inserter.close();
+ writer.close();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ pigServer.shutdown();
+ }
+
+ @Test
+ public void test_union_empty_left_table() throws ExecException, IOException {
+ //
+ // Test sorted union with two tables and left table is empty
+ //
+
+ // Sort tables
+ String orderby1 = "sort1 = ORDER table1 BY " + "a,b" + " ;";
+ pigServer.registerQuery(orderby1);
+
+ String orderby2 = "sort2 = ORDER table2 BY " + "a,b" + " ;";
+ pigServer.registerQuery(orderby2);
+
+ // Filter to remove all rows in table
+ String filter = "filtersort1 = FILTER sort1 BY a == -99 ;";
+
+ pigServer.registerQuery(filter);
+
+ Path newPath = new Path(getCurrentMethodName());
+
+ // Store sorted tables
+ String pathSort1 = newPath.toString() + "1";
+ pigJob = pigServer.store("filtersort1", pathSort1, TableStorer.class.getCanonicalName() +
+ "('" + TABLE1_STORAGE + "')");
+ Assert.assertNull(pigJob.getException());
+
+ String pathSort2 = newPath.toString() + "2";
+ pigJob = pigServer.store("sort2", pathSort2, TableStorer.class.getCanonicalName() +
+ "('" + TABLE2_STORAGE + "')");
+ Assert.assertNull(pigJob.getException());
+
+ String queryLoad = "records1 = LOAD '"
+ + pathSort1 + ","
+ + pathSort2
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader('a,source_table,b,c,d,e,f,m1', 'sorted');";
+
+ System.out.println("queryLoad: " + queryLoad);
+
+ pigServer.registerQuery(queryLoad);
+
+ // Verify union table
+ ArrayList<ArrayList<Object>> resultTable = new ArrayList<ArrayList<Object>>();
+
+ addResultRow(resultTable, -1, 1, -99.0f, 1002L, 51e+2, "orange", new DataByteArray("orange"),m2);
+ addResultRow(resultTable, 15, 1, 56.0f, 1004L, 50e+2, "green", new DataByteArray("green"), m2);
+ addResultRow(resultTable, 1001, 1, 100.0f, 1003L, 55e+2, "white", new DataByteArray("white"), m2);
+ addResultRow(resultTable, 1001, 1, 102.0f, 1001L, 52e+2, "purple", new DataByteArray("purple"),m2);
+
+ addResultRow(resultTable, 1001, 1, 50.0f, 1008L, 52e+2, "gray", new DataByteArray("gray"), m2);
+ addResultRow(resultTable, 1001, 1, 53.0f, 1001L, 52e+2, "brown", new DataByteArray("brown"), m2);
+ addResultRow(resultTable, 2000, 1, 33.0f, 1006L, 52e+2, "beige", new DataByteArray("beige"), m2);
+
+ // Verify union table
+ Iterator<Tuple> it = pigServer.openIterator("records1");
+ int numbRows = verifyTable(resultTable, 0, it);
+
+ Assert.assertEquals(numbRows, table2.length);
+ }
+
+ @Test
+ public void test_union_empty_right_table() throws ExecException, IOException {
+ //
+ // Test sorted union with two tables and right table is empty
+ //
+
+ // Sort tables
+ String orderby1 = "sort1 = ORDER table1 BY " + "e" + " ;";
+ pigServer.registerQuery(orderby1);
+
+ String orderby4 = "sort4 = ORDER table4 BY " + "e" + " ;";
+ pigServer.registerQuery(orderby4);
+
+ // Filter to remove all rows in table
+ String filter = "filtersort4 = FILTER sort4 BY e == 'teststring' ;";
+
+ pigServer.registerQuery(filter);
+
+ Path newPath = new Path(getCurrentMethodName());
+
+ // Store sorted tables
+ String pathSort1 = newPath.toString() + "1";
+ pigJob = pigServer.store("sort1", pathSort1, TableStorer.class.getCanonicalName() +
+ "('" + TABLE1_STORAGE + "')");
+ Assert.assertNull(pigJob.getException());
+
+ String pathSort4 = newPath.toString() + "4";
+ pigJob = pigServer.store("filtersort4", pathSort4, TableStorer.class.getCanonicalName() +
+ "('" + TABLE4_STORAGE + "')");
+ Assert.assertNull(pigJob.getException());
+
+ String queryLoad = "records1 = LOAD '"
+ + pathSort1 + ","
+ + pathSort4
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader('e,source_table,a,b,c,d,f,m1,str3,str4', 'sorted');";
+
+ System.out.println("queryLoad: " + queryLoad);
+
+ pigServer.registerQuery(queryLoad);
+
+ // Verify union table
+ ArrayList<ArrayList<Object>> resultTable = new ArrayList<ArrayList<Object>>();
+
+ addResultRow(resultTable, "Apple", 0, 1001, 100.0f, 1003L, 50e+2, new DataByteArray("Apple"), m1, null, null);
+ addResultRow(resultTable, "Hadoop", 0, 1002, 28.0f, 1000L, 50e+2, new DataByteArray("Hadoop"),m1, null, null);
+ addResultRow(resultTable, "Pig", 0, 1001, 50.0f, 1000L, 50e+2, new DataByteArray("Pig"), m1, null, null);
+ addResultRow(resultTable, "Zebra", 0, 5, -3.25f, 1001L, 51e+2, new DataByteArray("Zebra"), m1, null, null);
+
+ addResultRow(resultTable, "apple", 0, 1001, 101.0f, 1001L, 50e+2, new DataByteArray("apple"), m1, null, null);
+ addResultRow(resultTable, "hadoop", 0, 1000, 0.0f, 1002L, 52e+2, new DataByteArray("hadoop"),m1, null, null);
+ addResultRow(resultTable, "pig", 0, 1001, 52.0f, 1001L, 50e+2, new DataByteArray("pig"), m1, null, null);
+ addResultRow(resultTable, "zebra", 0, -1, 3.25f, 1000L, 50e+2, new DataByteArray("zebra"), m1, null, null);
+
+ // Verify union table
+ Iterator<Tuple> it = pigServer.openIterator("records1");
+ int numbRows = verifyTable(resultTable, 0, it);
+
+ Assert.assertEquals(numbRows, table1.length);
+ }
+
+ @Test
+ public void test_union_empty_middle_table() throws ExecException, IOException {
+ //
+ // Test sorted union with three tables and middle table is empty
+ //
+
+ // Sort tables
+ String orderby1 = "sort1 = ORDER table1 BY " + "e" + " ;";
+ pigServer.registerQuery(orderby1);
+
+ String orderby2 = "sort2 = ORDER table2 BY " + "e" + " ;";
+ pigServer.registerQuery(orderby2);
+
+ String orderby4 = "sort4 = ORDER table4 BY " + "e" + " ;";
+ pigServer.registerQuery(orderby4);
+
+ // Filter to remove all rows in table
+ String filter = "filtersort2 = FILTER sort2 BY e == 'teststring' ;";
+
+ pigServer.registerQuery(filter);
+
+ Path newPath = new Path(getCurrentMethodName());
+
+ // Store sorted tables
+ String pathSort1 = newPath.toString() + "1";
+ pigJob = pigServer.store("sort1", pathSort1, TableStorer.class.getCanonicalName() +
+ "('" + TABLE1_STORAGE + "')");
+ Assert.assertNull(pigJob.getException());
+
+ String pathSort2 = newPath.toString() + "2";
+ pigJob = pigServer.store("filtersort2", pathSort2, TableStorer.class.getCanonicalName() +
+ "('" + TABLE2_STORAGE + "')");
+ Assert.assertNull(pigJob.getException());
+
+ String pathSort4 = newPath.toString() + "4";
+ pigJob = pigServer.store("sort4", pathSort4, TableStorer.class.getCanonicalName() +
+ "('" + TABLE4_STORAGE + "')");
+ Assert.assertNull(pigJob.getException());
+
+ String queryLoad = "records1 = LOAD '"
+ + pathSort1 + ","
+ + pathSort2 + ","
+ + pathSort4
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader('e,source_table,a,b,c,d,f,m1,str3,str4', 'sorted');";
+
+ System.out.println("queryLoad: " + queryLoad);
+
+ pigServer.registerQuery(queryLoad);
+
+ // Verify union table
+ ArrayList<ArrayList<Object>> resultTable = new ArrayList<ArrayList<Object>>();
+
+ addResultRow(resultTable, "Apple", 0, 1001, 100.0f, 1003L, 50e+2, new DataByteArray("Apple"), m1, null, null);
+ addResultRow(resultTable, "Hadoop", 0, 1002, 28.0f, 1000L, 50e+2, new DataByteArray("Hadoop"),m1, null, null);
+ addResultRow(resultTable, "Pig", 0, 1001, 50.0f, 1000L, 50e+2, new DataByteArray("Pig"), m1, null, null);
+ addResultRow(resultTable, "Zebra", 0, 5, -3.25f, 1001L, 51e+2, new DataByteArray("Zebra"), m1, null, null);
+
+ addResultRow(resultTable, "a 1", 2, null, null, null, null, null, null, "Reno", "Nevada");
+ addResultRow(resultTable, "a 2", 2, null, null, null, null, null, null, "Dallas", "Texas");
+ addResultRow(resultTable, "a 3", 2, null, null, null, null, null, null, "Phoenix", "Arizona");
+ addResultRow(resultTable, "a 4", 2, null, null, null, null, null, null, "New York", "New York");
+
+ addResultRow(resultTable, "a 5", 2, null, null, null, null, null, null, "Las Vegas","Nevada");
+ addResultRow(resultTable, "a 6", 2, null, null, null, null, null, null, "Santa Cruz","California");
+ addResultRow(resultTable, "a 7", 2, null, null, null, null, null, null, "San Jose", "California");
+ addResultRow(resultTable, "a 8", 2, null, null, null, null, null, null, "Cupertino","California");
+
+ addResultRow(resultTable, "apple", 0, 1001, 101.0f, 1001L, 50e+2, new DataByteArray("apple"), m1, null, null);
+ addResultRow(resultTable, "hadoop", 0, 1000, 0.0f, 1002L, 52e+2, new DataByteArray("hadoop"),m1, null, null);
+ addResultRow(resultTable, "pig", 0, 1001, 52.0f, 1001L, 50e+2, new DataByteArray("pig"), m1, null, null);
+ addResultRow(resultTable, "zebra", 0, -1, 3.25f, 1000L, 50e+2, new DataByteArray("zebra"), m1, null, null);
+
+ // Verify union table
+ Iterator<Tuple> it = pigServer.openIterator("records1");
+ int numbRows = verifyTable(resultTable, 0, it);
+
+ Assert.assertEquals(numbRows, table1.length + table4.length);
+ }
+
+ @Test
+ public void test_union_empty_many_table() throws ExecException, IOException {
+ //
+ // Test sorted union with three tables with left and right table empty
+ //
+
+ // Sort tables
+ String orderby1 = "sort1 = ORDER table1 BY " + "e" + " ;";
+ pigServer.registerQuery(orderby1);
+
+ String orderby2 = "sort2 = ORDER table2 BY " + "e" + " ;";
+ pigServer.registerQuery(orderby2);
+
+ String orderby4 = "sort4 = ORDER table4 BY " + "e" + " ;";
+ pigServer.registerQuery(orderby4);
+
+ // Filter to remove all rows in table
+ String filter1 = "filtersort1 = FILTER sort1 BY e == 'teststring' ;";
+ pigServer.registerQuery(filter1);
+
+ // Filter to remove all rows in table
+ String filter4 = "filtersort4 = FILTER sort4 BY e == 'teststring' ;";
+ pigServer.registerQuery(filter4);
+
+ Path newPath = new Path(getCurrentMethodName());
+
+ // Store sorted tables
+ String pathSort1 = newPath.toString() + "1";
+ pigJob = pigServer.store("filtersort1", pathSort1, TableStorer.class.getCanonicalName() +
+ "('" + TABLE1_STORAGE + "')");
+ Assert.assertNull(pigJob.getException());
+
+ String pathSort2 = newPath.toString() + "2";
+ pigJob = pigServer.store("sort2", pathSort2, TableStorer.class.getCanonicalName() +
+ "('" + TABLE2_STORAGE + "')");
+ Assert.assertNull(pigJob.getException());
+
+ String pathSort4 = newPath.toString() + "4";
+ pigJob = pigServer.store("filtersort4", pathSort4, TableStorer.class.getCanonicalName() +
+ "('" + TABLE4_STORAGE + "')");
+ Assert.assertNull(pigJob.getException());
+
+ String queryLoad = "records1 = LOAD '"
+ + pathSort1 + ","
+ + pathSort2 + ","
+ + pathSort4
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader('e,source_table,a,b,c,d,f,m1,str3,str4', 'sorted');";
+
+ System.out.println("queryLoad: " + queryLoad);
+
+ pigServer.registerQuery(queryLoad);
+
+ // Verify union table
+ ArrayList<ArrayList<Object>> resultTable = new ArrayList<ArrayList<Object>>();
+
+ addResultRow(resultTable, "beige", 1, 2000, 33.0f, 1006L, 52e+2, new DataByteArray("beige"), m2, null, null);
+ addResultRow(resultTable, "brown", 1, 1001, 53.0f, 1001L, 52e+2, new DataByteArray("brown"), m2, null, null);
+ addResultRow(resultTable, "gray", 1, 1001, 50.0f, 1008L, 52e+2, new DataByteArray("gray"), m2, null, null);
+ addResultRow(resultTable, "green", 1, 15, 56.0f, 1004L, 50e+2, new DataByteArray("green"), m2, null, null);
+
+ addResultRow(resultTable, "orange", 1, -1, -99.0f, 1002L, 51e+2, new DataByteArray("orange"),m2, null, null);
+ addResultRow(resultTable, "purple", 1, 1001, 102.0f, 1001L, 52e+2, new DataByteArray("purple"),m2, null, null);
+ addResultRow(resultTable, "white", 1, 1001, 100.0f, 1003L, 55e+2, new DataByteArray("white"), m2, null, null);
+
+ // Verify union table
+ Iterator<Tuple> it = pigServer.openIterator("records1");
+ int numbRows = verifyTable(resultTable, 0, it);
+
+ Assert.assertEquals(numbRows, table2.length);
+ }
+
+ @Test
+ public void test_union_empty_all_table() throws ExecException, IOException {
+ //
+ // Test sorted union with two tables and both are empty
+ //
+
+ // Sort tables
+ String orderby1 = "sort1 = ORDER table1 BY " + "e" + " ;";
+ pigServer.registerQuery(orderby1);
+
+ String orderby4 = "sort4 = ORDER table4 BY " + "e" + " ;";
+ pigServer.registerQuery(orderby4);
+
+ // Filter to remove all rows in table
+ String filter1 = "filtersort1 = FILTER sort1 BY e == 'teststring' ;";
+ pigServer.registerQuery(filter1);
+
+ // Filter to remove all rows in table
+ String filter4 = "filtersort4 = FILTER sort4 BY e == 'teststring' ;";
+ pigServer.registerQuery(filter4);
+
+ Path newPath = new Path(getCurrentMethodName());
+
+ // Store sorted tables
+ String pathSort1 = newPath.toString() + "1";
+ pigJob = pigServer.store("filtersort1", pathSort1, TableStorer.class.getCanonicalName() +
+ "('" + TABLE1_STORAGE + "')");
+ Assert.assertNull(pigJob.getException());
+
+ String pathSort4 = newPath.toString() + "4";
+ pigJob = pigServer.store("filtersort4", pathSort4, TableStorer.class.getCanonicalName() +
+ "('" + TABLE4_STORAGE + "')");
+ Assert.assertNull(pigJob.getException());
+
+ String queryLoad = "records1 = LOAD '"
+ + pathSort1 + ","
+ + pathSort4
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader('e,source_table,a,b,c,d,f,m1,str3,str4', 'sorted');";
+
+ System.out.println("queryLoad: " + queryLoad);
+
+ pigServer.registerQuery(queryLoad);
+
+ int numbRows = printTable("records1");
+
+ Assert.assertEquals(numbRows, 0);
+ }
+
+ @Test
+ public void test_pig_statements() throws ExecException, IOException {
+ //
+ // Test sorted union with two tables after pig filter and pig foreach statements
+ //
+
+ // Sort tables
+ String orderby1 = "sort1 = ORDER table1 BY " + "e" + " ;";
+ pigServer.registerQuery(orderby1);
+
+ String orderby4 = "sort4 = ORDER table4 BY " + "e" + " ;";
+ pigServer.registerQuery(orderby4);
+
+ // Table after pig filter
+ String filter1 = "filtersort1 = FILTER sort1 BY a == 1001 ;";
+ pigServer.registerQuery(filter1);
+
+ Path newPath = new Path(getCurrentMethodName());
+
+ // Store sorted tables
+ String pathSort1 = newPath.toString() + "1";
+ pigJob = pigServer.store("filtersort1", pathSort1, TableStorer.class.getCanonicalName() +
+ "('" + TABLE1_STORAGE + "')");
+ Assert.assertNull(pigJob.getException());
+
+ String pathSort4 = newPath.toString() + "4";
+ pigJob = pigServer.store("sort4", pathSort4, TableStorer.class.getCanonicalName() +
+ "('" + TABLE4_STORAGE + "')");
+ Assert.assertNull(pigJob.getException());
+
+ String queryLoad = "records1 = LOAD '"
+ + pathSort1 + ","
+ + pathSort4
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader('e,source_table,a,b,c,d,f,m1,str3,str4', 'sorted');";
+
+ System.out.println("queryLoad: " + queryLoad);
+
+ pigServer.registerQuery(queryLoad);
+
+ // Verify union table
+ ArrayList<ArrayList<Object>> resultTable = new ArrayList<ArrayList<Object>>();
+
+ addResultRow(resultTable, "Apple", 0, 1001, 100.0f, 1003L, 50e+2, new DataByteArray("Apple"), m1, null, null);
+ addResultRow(resultTable, "Pig", 0, 1001, 50.0f, 1000L, 50e+2, new DataByteArray("Pig"), m1, null, null);
+ addResultRow(resultTable, "a 1", 1, null, null, null, null, null, null, "Reno", "Nevada");
+ addResultRow(resultTable, "a 2", 1, null, null, null, null, null, null, "Dallas", "Texas");
+
+ addResultRow(resultTable, "a 3", 1, null, null, null, null, null, null, "Phoenix", "Arizona");
+ addResultRow(resultTable, "a 4", 1, null, null, null, null, null, null, "New York", "New York");
+ addResultRow(resultTable, "a 5", 1, null, null, null, null, null, null, "Las Vegas","Nevada");
+ addResultRow(resultTable, "a 6", 1, null, null, null, null, null, null, "Santa Cruz","California");
+
+ addResultRow(resultTable, "a 7", 1, null, null, null, null, null, null, "San Jose", "California");
+ addResultRow(resultTable, "a 8", 1, null, null, null, null, null, null, "Cupertino","California");
+ addResultRow(resultTable, "apple", 0, 1001, 101.0f, 1001L, 50e+2, new DataByteArray("apple"), m1, null, null);
+ addResultRow(resultTable, "pig", 0, 1001, 52.0f, 1001L, 50e+2, new DataByteArray("pig"), m1, null, null);
+
+ // Verify union table
+ Iterator<Tuple> it = pigServer.openIterator("records1");
+ int numbRows = verifyTable(resultTable, 0, it);
+
+ Assert.assertEquals(12, numbRows);
+ }
+
+ @Test
+ public void test_union_as_input() throws ExecException, IOException {
+ //
+ // Test sorted union with two tables, one of the tables is from previous sorted union
+ //
+
+ // Sort tables
+ String orderby1 = "sort1 = ORDER table1 BY " + "e" + " ;";
+ pigServer.registerQuery(orderby1);
+
+ String orderby2 = "sort2 = ORDER table2 BY " + "e" + " ;";
+ pigServer.registerQuery(orderby2);
+
+ String orderby4 = "sort4 = ORDER table4 BY " + "e" + " ;";
+ pigServer.registerQuery(orderby4);
+
+ Path newPath = new Path(getCurrentMethodName());
+
+ // Store sorted tables
+ String pathSort1 = newPath.toString() + ++fileId;
+ pigJob = pigServer.store("sort1", pathSort1, TableStorer.class.getCanonicalName() +
+ "('" + TABLE1_STORAGE + "')");
+ Assert.assertNull(pigJob.getException());
+
+ String pathSort2 = newPath.toString() + ++fileId;
+ pigJob = pigServer.store("sort2", pathSort2, TableStorer.class.getCanonicalName() +
+ "('" + TABLE2_STORAGE + "')");
+ Assert.assertNull(pigJob.getException());
+
+ String pathSort4 = newPath.toString() + ++fileId;
+ pigJob = pigServer.store("sort4", pathSort4, TableStorer.class.getCanonicalName() +
+ "('" + TABLE4_STORAGE + "')");
+ Assert.assertNull(pigJob.getException());
+
+ // Create first sorted union
+ String queryLoad1 = "records1 = LOAD '"
+ + pathSort1 + ","
+ + pathSort2
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader('e,a,b,c,d,f,m1', 'sorted');";
+ pigServer.registerQuery(queryLoad1);
+
+ // Add sort info to union table before store
+ String sortunion = "sortrecords1 = ORDER records1 BY " + "e" + " ;";
+ pigServer.registerQuery(sortunion);
+
+ // Store union table
+ String pathUnion1 = newPath.toString() + ++fileId;
+ pigJob = pigServer.store("sortrecords1", pathUnion1, TableStorer.class.getCanonicalName() +
+ "('" + TABLE1_STORAGE + "')");
+ Assert.assertNull(pigJob.getException());
+
+ // Create second sorted union
+ String queryLoad2 = "records2 = LOAD '"
+ + pathUnion1 + ","
+ + pathSort4
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader('e,source_table,a,b,str3,str4', 'sorted');";
+
+ System.out.println("queryLoad: " + queryLoad2);
+ pigServer.registerQuery(queryLoad2);
+
+ // Verify union table
+ ArrayList<ArrayList<Object>> resultTable = new ArrayList<ArrayList<Object>>();
+
+ addResultRow(resultTable, "Apple", 0, 1001, 100.0f, null, null);
+ addResultRow(resultTable, "Hadoop", 0, 1002, 28.0f, null, null);
+ addResultRow(resultTable, "Pig", 0, 1001, 50.0f, null, null);
+ addResultRow(resultTable, "Zebra", 0, 5, -3.25f, null, null);
+
+ addResultRow(resultTable, "a 1", 1, null, null, "Reno", "Nevada");
+ addResultRow(resultTable, "a 2", 1, null, null, "Dallas", "Texas");
+ addResultRow(resultTable, "a 3", 1, null, null, "Phoenix", "Arizona");
+ addResultRow(resultTable, "a 4", 1, null, null, "New York", "New York");
+
+ addResultRow(resultTable, "a 5", 1, null, null, "Las Vegas","Nevada");
+ addResultRow(resultTable, "a 6", 1, null, null, "Santa Cruz","California");
+ addResultRow(resultTable, "a 7", 1, null, null, "San Jose","California");
+ addResultRow(resultTable, "a 8", 1, null, null, "Cupertino","California");
+
+ addResultRow(resultTable, "apple", 0, 1001, 101.0f, null, null);
+ addResultRow(resultTable, "beige", 0, 2000, 33.0f, null, null);
+ addResultRow(resultTable, "brown", 0, 1001, 53.0f, null, null);
+ addResultRow(resultTable, "gray", 0, 1001, 50.0f, null, null);
+
+ addResultRow(resultTable, "green", 0, 15, 56.0f, null, null);
+ addResultRow(resultTable, "hadoop", 0, 1000, 0.0f, null, null);
+ addResultRow(resultTable, "orange", 0, -1, -99.0f, null, null);
+ addResultRow(resultTable, "pig", 0, 1001, 52.0f, null, null);
+
+ addResultRow(resultTable, "purple", 0, 1001, 102.0f, null, null);
+ addResultRow(resultTable, "white", 0, 1001, 100.0f, null, null);
+ addResultRow(resultTable, "zebra", 0, -1, 3.25f, null, null);
+
+ // Verify union table
+ Iterator<Tuple> it = pigServer.openIterator("records2");
+ int numbRows = verifyTable(resultTable, 0, it);
+
+ Assert.assertEquals(23, numbRows);
+ }
+
+ /**
+ * Verify union output table with expected results
+ *
+ */
+ private int verifyTable(ArrayList<ArrayList<Object>> resultTable, int keyColumn, Iterator<Tuple> it) throws IOException {
+ int numbRows = 0;
+ int index = 0;
+ Object value = resultTable.get(index).get(keyColumn); // get value of primary key
+
+ while (it.hasNext()) {
+ Tuple rowValues = it.next();
+
+ // If last primary sort key does match then search for next matching key
+ if (! compareObj(value, rowValues.get(keyColumn))) {
+ int subIndex = index + 1;
+ while (subIndex < resultTable.size()) {
+ if ( ! compareObj(value, resultTable.get(subIndex).get(keyColumn)) ) { // found new key
+ index = subIndex;
+ value = resultTable.get(index).get(keyColumn);
+ break;
+ }
+ ++subIndex;
+ }
+ Assert.assertEquals("Table comparison error for row : " + numbRows + " - no key found for : "
+ + rowValues.get(keyColumn), value, rowValues.get(keyColumn));
+ }
+ // Search for matching row with this primary key
+ int subIndex = index;
+
+ while (subIndex < resultTable.size()) {
+ // Compare row
+ ArrayList<Object> resultRow = resultTable.get(subIndex);
+ if ( compareRow(rowValues, resultRow) )
+ break; // found matching row
+ ++subIndex;
+ Assert.assertEquals("Table comparison error for row : " + numbRows + " - no matching row found for : "
+ + rowValues.get(keyColumn), value, resultTable.get(subIndex).get(keyColumn));
+ }
+ ++numbRows;
+ }
+ Assert.assertEquals(resultTable.size(), numbRows); // verify expected row count
+ return numbRows;
+ }
+
+ /**
+ * Compare table rows
+ *
+ */
+ private boolean compareRow(Tuple rowValues, ArrayList<Object> resultRow) throws IOException {
+ boolean result = true;
+ Assert.assertEquals(resultRow.size(), rowValues.size());
+ for (int i = 0; i < rowValues.size(); ++i) {
+ if (! compareObj(rowValues.get(i), resultRow.get(i)) ) {
+ result = false;
+ break;
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Compare table values
+ *
+ */
+ private boolean compareObj(Object object1, Object object2) {
+ if (object1 == null) {
+ if (object2 == null)
+ return true;
+ else
+ return false;
+ } else if (object1.equals(object2))
+ return true;
+ else
+ return false;
+ }
+
+ /**
+ *Add a row to expected results table
+ *
+ */
+ private void addResultRow(ArrayList<ArrayList<Object>> resultTable, Object ... values) {
+ ArrayList<Object> resultRow = new ArrayList<Object>();
+
+ for (int i = 0; i < values.length; i++) {
+ resultRow.add(values[i]);
+ }
+ resultTable.add(resultRow);
+ }
+
+ /**
+ * Print Pig Table (for debugging)
+ *
+ */
+ private int printTable(String tablename) throws IOException {
+ Iterator<Tuple> it1 = pigServer.openIterator(tablename);
+ int numbRows = 0;
+ while (it1.hasNext()) {
+ Tuple RowValue1 = it1.next();
+ ++numbRows;
+ System.out.println();
+ for (int i = 0; i < RowValue1.size(); ++i)
+ System.out.println("DEBUG: " + tablename + " RowValue.get(" + i + ") = " + RowValue1.get(i));
+ }
+ System.out.println("\nRow count : " + numbRows);
+ return numbRows;
+ }
+
+ /**
+ * Return the name of the routine that called getCurrentMethodName
+ *
+ */
+ private String getCurrentMethodName() {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ (new Throwable()).printStackTrace(pw);
+ pw.flush();
+ String stackTrace = baos.toString();
+ pw.close();
+
+ StringTokenizer tok = new StringTokenizer(stackTrace, "\n");
+ tok.nextToken(); // 'java.lang.Throwable'
+ tok.nextToken(); // 'at ...getCurrentMethodName'
+ String l = tok.nextToken(); // 'at ...<caller to getCurrentRoutine>'
+ // Parse line 3
+ tok = new StringTokenizer(l.trim(), " <(");
+ String t = tok.nextToken(); // 'at'
+ t = tok.nextToken(); // '...<caller to getCurrentRoutine>'
+ return t;
+ }
+
+}
Modified: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestSimpleType.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestSimpleType.java?rev=883836&r1=883835&r2=883836&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestSimpleType.java (original)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestSimpleType.java Tue Nov 24 19:54:19 2009
@@ -100,7 +100,7 @@
System.out.println("path =" + path);
BasicTable.Writer writer = new BasicTable.Writer(path, STR_SCHEMA,
- STR_STORAGE, false, conf);
+ STR_STORAGE, conf);
Schema schema = writer.getSchema();
BasicTable.Writer writer1 = new BasicTable.Writer(path, conf);
@@ -165,7 +165,7 @@
t = tok.nextToken(); // '...<caller to getCurrentRoutine>'
return t;
}
-
+
// @Test
public void testReadSimpleStitch() throws IOException, ParseException {
String query = "records = LOAD '" + path.toString()
@@ -242,9 +242,9 @@
@Test
// Store same table
public void testStorer() throws ExecException, IOException {
- /*
- * Use pig LOAD to load testing data for store
- */
+ //
+ // Use pig LOAD to load testing data for store
+ //
String query = "records = LOAD '"
+ path.toString()
+ "' USING org.apache.hadoop.zebra.pig.TableLoader() as (s1,s2,s3,s4,s5,s6);";
@@ -256,9 +256,9 @@
System.out.println(RowValue);
}
- /*
- * Use pig STORE to store testing data
- */
+ //
+ // Use pig STORE to store testing data
+ //
Path newPath = new Path(getCurrentMethodName());
pigServer
.store(
@@ -528,19 +528,19 @@
@Test
// Store negative, store to same path. Store should fail
- public void testStorer5() throws ExecException, IOException {
- /*
- * Use pig LOAD to load testing data for store
- */
+ public void testStorer5() throws ExecException, IOException {
+ //
+ // Use pig LOAD to load testing data for store
+ //
String query = "records = LOAD '"
+ path.toString()
+ "' USING org.apache.hadoop.zebra.pig.TableLoader() as (s1,s2,s3,s4,s5,s6);";
pigServer.registerQuery(query);
- /*
- * Use pig STORE to store testing data
- */
-
+ //
+ // Use pig STORE to store testing data
+ //
+ System.out.println("path = " + path);
ExecJob pigJob = pigServer
.store(
"records",
@@ -548,7 +548,6 @@
TableStorer.class.getCanonicalName()
+ "('[s1, s2]; [s3, s4]')");
Assert.assertNotNull(pigJob.getException());
- System.out.println(pigJob.getException());
+ System.out.println("pig job exception : " + pigJob.getException());
}
-
}
Added: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestSortedTableUnion.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestSortedTableUnion.java?rev=883836&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestSortedTableUnion.java (added)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestSortedTableUnion.java Tue Nov 24 19:54:19 2009
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.zebra.pig;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.StringTokenizer;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.pig.TableStorer;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.test.MiniCluster;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Note:
+ *
+ * Make sure you add the build/pig-0.1.0-dev-core.jar to the Classpath of the
+ * app/debug configuration, when run this from inside the Eclipse.
+ *
+ */
+public class TestSortedTableUnion {
+ protected static ExecType execType = ExecType.MAPREDUCE;
+ private static MiniCluster cluster;
+ protected static PigServer pigServer;
+ private static Path pathTable;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ if (System.getProperty("hadoop.log.dir") == null) {
+ String base = new File(".").getPath(); // getAbsolutePath();
+ System
+ .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
+ }
+
+ if (execType == ExecType.MAPREDUCE) {
+ cluster = MiniCluster.buildCluster();
+ pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ } else {
+ pigServer = new PigServer(ExecType.LOCAL);
+ }
+
+ Configuration conf = new Configuration();
+ FileSystem fs = cluster.getFileSystem();
+ Path pathWorking = fs.getWorkingDirectory();
+ pathTable = new Path(pathWorking, "TestTableStorer");
+ System.out.println("pathTable =" + pathTable);
+ BasicTable.Writer writer = new BasicTable.Writer(pathTable,
+ "SF_a:string,SF_b:string,SF_c,SF_d,SF_e,SF_f,SF_g",
+ "[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]", conf);
+ Schema schema = writer.getSchema();
+ Tuple tuple = TypesUtils.createTuple(schema);
+
+ final int numsBatch = 10;
+ final int numsInserters = 1;
+ TableInserter[] inserters = new TableInserter[numsInserters];
+ for (int i = 0; i < numsInserters; i++) {
+ inserters[i] = writer.getInserter("ins" + i, false);
+ }
+
+ for (int b = 0; b < numsBatch; b++) {
+ for (int i = 0; i < numsInserters; i++) {
+ TypesUtils.resetTuple(tuple);
+ for (int k = 0; k < tuple.size(); ++k) {
+ try {
+ tuple.set(k, (9-b) + "_" + i + "" + k);
+ } catch (ExecException e) {
+ e.printStackTrace();
+ }
+ }
+ inserters[i].insert(new BytesWritable(("key" + i).getBytes()), tuple);
+ }
+ }
+ for (int i = 0; i < numsInserters; i++) {
+ inserters[i].close();
+ }
+ writer.close();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ pigServer.shutdown();
+ }
+
+ /**
+ * Return the name of the routine that called getCurrentMethodName
+ *
+ */
+ public String getCurrentMethodName() {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ (new Throwable()).printStackTrace(pw);
+ pw.flush();
+ String stackTrace = baos.toString();
+ pw.close();
+
+ StringTokenizer tok = new StringTokenizer(stackTrace, "\n");
+ tok.nextToken(); // 'java.lang.Throwable'
+ tok.nextToken(); // 'at ...getCurrentMethodName'
+ String l = tok.nextToken(); // 'at ...<caller to getCurrentRoutine>'
+ // Parse line 3
+ tok = new StringTokenizer(l.trim(), " <(");
+ String t = tok.nextToken(); // 'at'
+ t = tok.nextToken(); // '...<caller to getCurrentRoutine>'
+ return t;
+ }
+
+ @Test
+ public void testStorer() throws ExecException, IOException {
+ /*
+ * Use pig LOAD to load testing data for store
+ */
+ String query = "records = LOAD '" + pathTable.toString()
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query);
+
+ /*
+ Iterator<Tuple> it2 = pigServer.openIterator("records");
+ int row0 = 0;
+ Tuple RowValue2 = null;
+ while (it2.hasNext()) {
+ // Last row value
+ RowValue2 = it2.next();
+ row0++;
+ if (row0 == 10) {
+ Assert.assertEquals("0_01", RowValue2.get(1));
+ Assert.assertEquals("0_00", RowValue2.get(0));
+ }
+ }
+ Assert.assertEquals(10, row0);
+ */
+
+ String orderby = "srecs = ORDER records BY SF_a;";
+ pigServer.registerQuery(orderby);
+
+ /*
+ * Use pig STORE to store testing data BasicTable.Writer writer = new
+ * BasicTable.Writer(pathTable, "SF_a,SF_b,SF_c,SF_d,SF_e,SF_f,SF_g",
+ * "[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]", false, conf);
+ */
+ Path newPath = new Path(getCurrentMethodName());
+
+ /*
+ * Table1 creation
+ */
+ pigServer
+ .store(
+ "srecs",
+ newPath.toString()+"1",
+ TableStorer.class.getCanonicalName()
+ + "('[SF_a, SF_b, SF_c]; [SF_e]')");
+ pigServer
+ .store(
+ "srecs",
+ newPath.toString()+"2",
+ TableStorer.class.getCanonicalName()
+ + "('[SF_a, SF_b, SF_c]; [SF_e]')");
+
+
+ String query4 = "records2 = LOAD '"
+ + newPath.toString() + "1,"
+ + newPath.toString() + "2"
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader('', 'sorted');";
+ pigServer.registerQuery(query4);
+
+ // check JOIN content
+ Iterator<Tuple> it3 = pigServer.openIterator("records2");
+ int row = 0, index;
+ Tuple RowValue3 = null;
+ while (it3.hasNext()) {
+ // Last row value
+ RowValue3 = it3.next();
+ Assert.assertEquals(7, RowValue3.size());
+ row++;
+ index = (row-1)/2;
+ Assert.assertEquals(index+"_01", RowValue3.get(1));
+ Assert.assertEquals(index+"_00", RowValue3.get(0));
+ Assert.assertEquals(index+"_06", RowValue3.get(6));
+ Assert.assertEquals(index+"_05", RowValue3.get(5));
+ Assert.assertEquals(index+"_04", RowValue3.get(4));
+ Assert.assertEquals(index+"_03", RowValue3.get(3));
+ Assert.assertEquals(index+"_02", RowValue3.get(2));
+ }
+ Assert.assertEquals(20, row);
+ }
+}
Added: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestSortedTableUnionMergeJoin.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestSortedTableUnionMergeJoin.java?rev=883836&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestSortedTableUnionMergeJoin.java (added)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestSortedTableUnionMergeJoin.java Tue Nov 24 19:54:19 2009
@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.zebra.pig;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.StringTokenizer;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.pig.TableStorer;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.test.MiniCluster;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Note:
+ *
+ * Make sure you add the build/pig-0.1.0-dev-core.jar to the Classpath of the
+ * app/debug configuration, when run this from inside the Eclipse.
+ *
+ */
+public class TestSortedTableUnionMergeJoin {
+ protected static ExecType execType = ExecType.MAPREDUCE;
+ private static MiniCluster cluster;
+ protected static PigServer pigServer;
+ private static Path pathTable;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ if (System.getProperty("hadoop.log.dir") == null) {
+ String base = new File(".").getPath(); // getAbsolutePath();
+ System
+ .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
+ }
+
+ if (execType == ExecType.MAPREDUCE) {
+ cluster = MiniCluster.buildCluster();
+ pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ } else {
+ pigServer = new PigServer(ExecType.LOCAL);
+ }
+
+ Configuration conf = new Configuration();
+ FileSystem fs = cluster.getFileSystem();
+ Path pathWorking = fs.getWorkingDirectory();
+ pathTable = new Path(pathWorking, "TestTableStorer");
+ System.out.println("pathTable =" + pathTable);
+ BasicTable.Writer writer = new BasicTable.Writer(pathTable,
+ "SF_a:string,SF_b:string,SF_c,SF_d,SF_e,SF_f,SF_g",
+ "[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]", conf);
+ Schema schema = writer.getSchema();
+ Tuple tuple = TypesUtils.createTuple(schema);
+
+ final int numsBatch = 10;
+ final int numsInserters = 1;
+ TableInserter[] inserters = new TableInserter[numsInserters];
+ for (int i = 0; i < numsInserters; i++) {
+ inserters[i] = writer.getInserter("ins" + i, false);
+ }
+
+ for (int b = 0; b < numsBatch; b++) {
+ for (int i = 0; i < numsInserters; i++) {
+ TypesUtils.resetTuple(tuple);
+ for (int k = 0; k < tuple.size(); ++k) {
+ try {
+ tuple.set(k, (9-b) + "_" + i + "" + k);
+ } catch (ExecException e) {
+ e.printStackTrace();
+ }
+ }
+ inserters[i].insert(new BytesWritable(("key" + i).getBytes()), tuple);
+ }
+ }
+ for (int i = 0; i < numsInserters; i++) {
+ inserters[i].close();
+ }
+ writer.close();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ pigServer.shutdown();
+ }
+
+ /**
+ * Return the name of the routine that called getCurrentMethodName
+ *
+ */
+ public String getCurrentMethodName() {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ (new Throwable()).printStackTrace(pw);
+ pw.flush();
+ String stackTrace = baos.toString();
+ pw.close();
+
+ StringTokenizer tok = new StringTokenizer(stackTrace, "\n");
+ tok.nextToken(); // 'java.lang.Throwable'
+ tok.nextToken(); // 'at ...getCurrentMethodName'
+ String l = tok.nextToken(); // 'at ...<caller to getCurrentRoutine>'
+ // Parse line 3
+ tok = new StringTokenizer(l.trim(), " <(");
+ String t = tok.nextToken(); // 'at'
+ t = tok.nextToken(); // '...<caller to getCurrentRoutine>'
+ return t;
+ }
+
+ @Test
+ public void testStorer() throws ExecException, IOException {
+ /*
+ * Use pig LOAD to load testing data for store
+ */
+ String query = "records = LOAD '" + pathTable.toString()
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query);
+
+ /*
+ Iterator<Tuple> it2 = pigServer.openIterator("records");
+ int row0 = 0;
+ Tuple RowValue2 = null;
+ while (it2.hasNext()) {
+ // Last row value
+ RowValue2 = it2.next();
+ row0++;
+ if (row0 == 10) {
+ Assert.assertEquals("0_01", RowValue2.get(1));
+ Assert.assertEquals("0_00", RowValue2.get(0));
+ }
+ }
+ Assert.assertEquals(10, row0);
+ */
+
+ String orderby = "srecs = ORDER records BY SF_a;";
+ pigServer.registerQuery(orderby);
+
+ /*
+ * Use pig STORE to store testing data BasicTable.Writer writer = new
+ * BasicTable.Writer(pathTable, "SF_a,SF_b,SF_c,SF_d,SF_e,SF_f,SF_g",
+ * "[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]", false, conf);
+ */
+ Path newPath = new Path(getCurrentMethodName());
+
+ /*
+ * Table1 creation
+ */
+ pigServer
+ .store(
+ "srecs",
+ newPath.toString()+"1",
+ TableStorer.class.getCanonicalName()
+ + "('[SF_a, SF_b, SF_c]; [SF_e]')");
+ pigServer
+ .store(
+ "srecs",
+ newPath.toString()+"2",
+ TableStorer.class.getCanonicalName()
+ + "('[SF_a, SF_b, SF_c]; [SF_e]')");
+
+ String query3 = "records1 = LOAD '"
+ + newPath.toString() + "1"
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader('', 'sorted');";
+ pigServer.registerQuery(query3);
+
+ String query4 = "records2 = LOAD '"
+ + newPath.toString() + "1,"
+ + newPath.toString() + "2"
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader('', 'sorted');";
+ pigServer.registerQuery(query4);
+
+ String join = "joinRecords = JOIN records1 BY SF_a, records2 BY SF_a USING \"merge\";";
+ pigServer.registerQuery(join);
+
+ // check JOIN content
+ Iterator<Tuple> it3 = pigServer.openIterator("records2");
+ int row = 0, index;
+ Tuple RowValue3 = null;
+ while (it3.hasNext()) {
+ // Last row value
+ RowValue3 = it3.next();
+ Assert.assertEquals(7, RowValue3.size());
+ row++;
+ index = (row-1)/2;
+ Assert.assertEquals(index+"_01", RowValue3.get(1));
+ Assert.assertEquals(index+"_00", RowValue3.get(0));
+ Assert.assertEquals(index+"_06", RowValue3.get(6));
+ Assert.assertEquals(index+"_05", RowValue3.get(5));
+ Assert.assertEquals(index+"_04", RowValue3.get(4));
+ Assert.assertEquals(index+"_03", RowValue3.get(3));
+ Assert.assertEquals(index+"_02", RowValue3.get(2));
+ }
+ Assert.assertEquals(20, row);
+ }
+}