You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by pr...@apache.org on 2009/11/24 20:54:34 UTC
svn commit: r883836 [12/23] - in /hadoop/pig/branches/load-store-redesign:
./ contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/
contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/
contrib/zebra/ contrib/zebr...
Modified: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoader.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoader.java?rev=883836&r1=883835&r2=883836&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoader.java (original)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoader.java Tue Nov 24 19:54:19 2009
@@ -76,7 +76,7 @@
System.out.println("pathTable =" + pathTable);
BasicTable.Writer writer = new BasicTable.Writer(pathTable,
- "a:string,b,c:string,d,e,f,g", "[a,b,c];[d,e,f,g]", false, conf);
+ "a:string,b,c:string,d,e,f,g", "[a,b,c];[d,e,f,g]", conf);
Schema schema = writer.getSchema();
Tuple tuple = TypesUtils.createTuple(schema);
Added: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoin.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoin.java?rev=883836&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoin.java (added)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoin.java Tue Nov 24 19:54:19 2009
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.zebra.pig;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.StringTokenizer;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.pig.TableStorer;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.test.MiniCluster;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Note:
+ *
+ * Make sure you add the build/pig-0.1.0-dev-core.jar to the Classpath of the
+ * app/debug configuration, when run this from inside the Eclipse.
+ *
+ */
+public class TestTableMergeJoin {
+ protected static ExecType execType = ExecType.MAPREDUCE;
+ private static MiniCluster cluster;
+ protected static PigServer pigServer;
+ private static Path pathTable;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ if (System.getProperty("hadoop.log.dir") == null) {
+ String base = new File(".").getPath(); // getAbsolutePath();
+ System
+ .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
+ }
+
+ if (execType == ExecType.MAPREDUCE) {
+ cluster = MiniCluster.buildCluster();
+ pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ } else {
+ pigServer = new PigServer(ExecType.LOCAL);
+ }
+
+ Configuration conf = new Configuration();
+ FileSystem fs = cluster.getFileSystem();
+ Path pathWorking = fs.getWorkingDirectory();
+ pathTable = new Path(pathWorking, "TestTableStorer");
+ System.out.println("pathTable =" + pathTable);
+ BasicTable.Writer writer = new BasicTable.Writer(pathTable,
+ "SF_a:string,SF_b:string,SF_c,SF_d,SF_e,SF_f,SF_g",
+ "[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]", conf);
+ Schema schema = writer.getSchema();
+ Tuple tuple = TypesUtils.createTuple(schema);
+
+ final int numsBatch = 10;
+ final int numsInserters = 1;
+ TableInserter[] inserters = new TableInserter[numsInserters];
+ for (int i = 0; i < numsInserters; i++) {
+ inserters[i] = writer.getInserter("ins" + i, false);
+ }
+
+ for (int b = 0; b < numsBatch; b++) {
+ for (int i = 0; i < numsInserters; i++) {
+ TypesUtils.resetTuple(tuple);
+ for (int k = 0; k < tuple.size(); ++k) {
+ try {
+ tuple.set(k, (9-b) + "_" + i + "" + k);
+ } catch (ExecException e) {
+ e.printStackTrace();
+ }
+ }
+ inserters[i].insert(new BytesWritable(("key" + i).getBytes()), tuple);
+ }
+ }
+ for (int i = 0; i < numsInserters; i++) {
+ inserters[i].close();
+ }
+ writer.close();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ pigServer.shutdown();
+ }
+
+ /**
+ * Return the name of the routine that called getCurrentMethodName
+ *
+ */
+ public String getCurrentMethodName() {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ (new Throwable()).printStackTrace(pw);
+ pw.flush();
+ String stackTrace = baos.toString();
+ pw.close();
+
+ StringTokenizer tok = new StringTokenizer(stackTrace, "\n");
+ tok.nextToken(); // 'java.lang.Throwable'
+ tok.nextToken(); // 'at ...getCurrentMethodName'
+ String l = tok.nextToken(); // 'at ...<caller to getCurrentRoutine>'
+ // Parse line 3
+ tok = new StringTokenizer(l.trim(), " <(");
+ String t = tok.nextToken(); // 'at'
+ t = tok.nextToken(); // '...<caller to getCurrentRoutine>'
+ return t;
+ }
+
+ @Test
+ public void testStorer() throws ExecException, IOException {
+ /*
+ * Use pig LOAD to load testing data for store
+ */
+ String query = "records = LOAD '" + pathTable.toString()
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query);
+
+ /*
+ Iterator<Tuple> it2 = pigServer.openIterator("records");
+ int row0 = 0;
+ Tuple RowValue2 = null;
+ while (it2.hasNext()) {
+ // Last row value
+ RowValue2 = it2.next();
+ row0++;
+ if (row0 == 10) {
+ Assert.assertEquals("0_01", RowValue2.get(1));
+ Assert.assertEquals("0_00", RowValue2.get(0));
+ }
+ }
+ Assert.assertEquals(10, row0);
+ */
+
+ String orderby = "srecs = ORDER records BY SF_a;";
+ pigServer.registerQuery(orderby);
+
+ /*
+ * Use pig STORE to store testing data BasicTable.Writer writer = new
+ * BasicTable.Writer(pathTable, "SF_a,SF_b,SF_c,SF_d,SF_e,SF_f,SF_g",
+ * "[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]", false, conf);
+ */
+ Path newPath = new Path(getCurrentMethodName());
+
+ /*
+ * Table1 creation
+ */
+ pigServer
+ .store(
+ "srecs",
+ newPath.toString()+"1",
+ TableStorer.class.getCanonicalName()
+ + "('[SF_a, SF_b, SF_c]; [SF_e]')");
+
+ String query3 = "records1 = LOAD '"
+ + newPath.toString() + "1"
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader('', 'sorted');";
+ pigServer.registerQuery(query3);
+
+ /*
+ Iterator<Tuple> it3 = pigServer.openIterator("records1");
+ int row = 0;
+ Tuple RowValue3 = null;
+ while (it3.hasNext()) {
+ // Last row value
+ RowValue3 = it3.next();
+ Assert.assertEquals(2, RowValue3.size());
+ row++;
+ if (row == 10) {
+ Assert.assertEquals("9_01", RowValue3.get(1));
+ Assert.assertEquals("9_00", RowValue3.get(0));
+ }
+ }
+ Assert.assertEquals(10, row);
+ */
+
+ /*
+ * Table2 creation
+ */
+ pigServer
+ .store(
+ "srecs",
+ newPath.toString()+"2",
+ TableStorer.class.getCanonicalName()
+ + "('[SF_a, SF_b, SF_c]; [SF_e]')");
+
+
+ String query4 = "records2 = LOAD '"
+ + newPath.toString() + "2"
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query4);
+
+ String join = "joinRecords = JOIN records1 BY SF_a, records2 BY SF_a USING \"merge\";";
+ pigServer.registerQuery(join);
+ // check JOIN content
+ Iterator<Tuple> it3 = pigServer.openIterator("joinRecords");
+ int row = 0;
+ Tuple RowValue3 = null;
+ while (it3.hasNext()) {
+ // Last row value
+ RowValue3 = it3.next();
+ Assert.assertEquals(14, RowValue3.size());
+ row++;
+ if (row == 10) {
+ Assert.assertEquals("9_01", RowValue3.get(1));
+ Assert.assertEquals("9_00", RowValue3.get(0));
+ Assert.assertEquals("9_01", RowValue3.get(8));
+ Assert.assertEquals("9_00", RowValue3.get(7));
+ Assert.assertEquals("9_06", RowValue3.get(6));
+ Assert.assertEquals("9_05", RowValue3.get(5));
+ Assert.assertEquals("9_04", RowValue3.get(4));
+ Assert.assertEquals("9_03", RowValue3.get(3));
+ Assert.assertEquals("9_02", RowValue3.get(2));
+ }
+ }
+ Assert.assertEquals(10, row);
+ }
+}
Added: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoinAfterFilter.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoinAfterFilter.java?rev=883836&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoinAfterFilter.java (added)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoinAfterFilter.java Tue Nov 24 19:54:19 2009
@@ -0,0 +1,253 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.zebra.pig;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.StringTokenizer;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.pig.TableStorer;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.test.MiniCluster;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Note:
+ *
+ * Make sure you add the build/pig-0.1.0-dev-core.jar to the Classpath of the
+ * app/debug configuration, when run this from inside the Eclipse.
+ *
+ */
+public class TestTableMergeJoinAfterFilter {
+ protected static ExecType execType = ExecType.MAPREDUCE;
+ private static MiniCluster cluster;
+ protected static PigServer pigServer;
+ private static Path pathTable;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ if (System.getProperty("hadoop.log.dir") == null) {
+ String base = new File(".").getPath(); // getAbsolutePath();
+ System
+ .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
+ }
+
+ if (execType == ExecType.MAPREDUCE) {
+ cluster = MiniCluster.buildCluster();
+ pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ } else {
+ pigServer = new PigServer(ExecType.LOCAL);
+ }
+
+ Configuration conf = new Configuration();
+ FileSystem fs = cluster.getFileSystem();
+ Path pathWorking = fs.getWorkingDirectory();
+ pathTable = new Path(pathWorking, "TestTableStorer");
+ System.out.println("pathTable =" + pathTable);
+ BasicTable.Writer writer = new BasicTable.Writer(pathTable,
+ "SF_a:string,SF_b:string,SF_c,SF_d,SF_e,SF_f,SF_g",
+ "[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]", conf);
+ Schema schema = writer.getSchema();
+ Tuple tuple = TypesUtils.createTuple(schema);
+
+ final int numsBatch = 10;
+ final int numsInserters = 1;
+ TableInserter[] inserters = new TableInserter[numsInserters];
+ for (int i = 0; i < numsInserters; i++) {
+ inserters[i] = writer.getInserter("ins" + i, false);
+ }
+
+ for (int b = 0; b < numsBatch; b++) {
+ for (int i = 0; i < numsInserters; i++) {
+ TypesUtils.resetTuple(tuple);
+ for (int k = 0; k < tuple.size(); ++k) {
+ try {
+ tuple.set(k, (9-b) + "_" + i + "" + k);
+ } catch (ExecException e) {
+ e.printStackTrace();
+ }
+ }
+ inserters[i].insert(new BytesWritable(("key" + i).getBytes()), tuple);
+ }
+ }
+ for (int i = 0; i < numsInserters; i++) {
+ inserters[i].close();
+ }
+ writer.close();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ pigServer.shutdown();
+ }
+
+ /**
+ * Return the name of the routine that called getCurrentMethodName
+ *
+ */
+ public String getCurrentMethodName() {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ (new Throwable()).printStackTrace(pw);
+ pw.flush();
+ String stackTrace = baos.toString();
+ pw.close();
+
+ StringTokenizer tok = new StringTokenizer(stackTrace, "\n");
+ tok.nextToken(); // 'java.lang.Throwable'
+ tok.nextToken(); // 'at ...getCurrentMethodName'
+ String l = tok.nextToken(); // 'at ...<caller to getCurrentRoutine>'
+ // Parse line 3
+ tok = new StringTokenizer(l.trim(), " <(");
+ String t = tok.nextToken(); // 'at'
+ t = tok.nextToken(); // '...<caller to getCurrentRoutine>'
+ return t;
+ }
+
+ @Test
+ public void testStorer() throws ExecException, IOException {
+ /*
+ * Use pig LOAD to load testing data for store
+ */
+ String query = "records = LOAD '" + pathTable.toString()
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query);
+
+ /*
+ Iterator<Tuple> it2 = pigServer.openIterator("records");
+ int row0 = 0;
+ Tuple RowValue2 = null;
+ while (it2.hasNext()) {
+ // Last row value
+ RowValue2 = it2.next();
+ row0++;
+ if (row0 == 10) {
+ Assert.assertEquals("0_01", RowValue2.get(1));
+ Assert.assertEquals("0_00", RowValue2.get(0));
+ }
+ }
+ Assert.assertEquals(10, row0);
+ */
+
+ String orderby = "srecs = ORDER records BY SF_a;";
+ pigServer.registerQuery(orderby);
+
+ /*
+ * Use pig STORE to store testing data BasicTable.Writer writer = new
+ * BasicTable.Writer(pathTable, "SF_a,SF_b,SF_c,SF_d,SF_e,SF_f,SF_g",
+ * "[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]", false, conf);
+ */
+ Path newPath = new Path(getCurrentMethodName());
+
+ /*
+ * Table1 creation
+ */
+ pigServer
+ .store(
+ "srecs",
+ newPath.toString()+"1",
+ TableStorer.class.getCanonicalName()
+ + "('[SF_a, SF_b, SF_c]; [SF_e]')");
+
+ String query3 = "records1 = LOAD '"
+ + newPath.toString() + "1"
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader('SF_a, SF_b', 'sorted');";
+ pigServer.registerQuery(query3);
+
+ /*
+ * Table2 creation
+ */
+ pigServer
+ .store(
+ "srecs",
+ newPath.toString()+"2",
+ TableStorer.class.getCanonicalName()
+ + "('[SF_a, SF_b, SF_c]; [SF_e]')");
+
+
+ String query4 = "records2 = LOAD '"
+ + newPath.toString() + "2"
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query4);
+
+ String filter = "records3 = FILTER records2 BY SF_a > '4';";
+ pigServer.registerQuery(filter);
+
+ Iterator<Tuple> it2 = pigServer.openIterator("records3");
+ int row2 = 0;
+ Tuple RowValue2 = null;
+ while (it2.hasNext()) {
+ // Last row value
+ RowValue2 = it2.next();
+ Assert.assertEquals(7, RowValue2.size());
+ row2++;
+ if (row2 == 5) {
+ Assert.assertEquals("8_01", RowValue2.get(1));
+ Assert.assertEquals("8_00", RowValue2.get(0));
+ }
+ }
+ Assert.assertEquals(6, row2);
+
+ String join = "joinRecords = JOIN records1 BY SF_a, records3 BY SF_a USING \"merge\";";
+ pigServer.registerQuery(join);
+ // check JOIN content
+ Iterator<Tuple> it3 = pigServer.openIterator("joinRecords");
+ int row = 0;
+ Tuple RowValue3 = null;
+ while (it3.hasNext()) {
+ // Last row value
+ RowValue3 = it3.next();
+ Assert.assertEquals(9, RowValue3.size());
+ row++;
+ if (row == 6) {
+ Assert.assertEquals("9_01", RowValue3.get(1));
+ Assert.assertEquals("9_00", RowValue3.get(0));
+ Assert.assertEquals("9_06", RowValue3.get(8));
+ Assert.assertEquals("9_05", RowValue3.get(7));
+ Assert.assertEquals("9_04", RowValue3.get(6));
+ Assert.assertEquals("9_03", RowValue3.get(5));
+ Assert.assertEquals("9_02", RowValue3.get(4));
+ Assert.assertEquals("9_01", RowValue3.get(3));
+ Assert.assertEquals("9_00", RowValue3.get(2));
+ }
+ }
+ Assert.assertEquals(6, row);
+ }
+}
Added: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoinFloat.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoinFloat.java?rev=883836&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoinFloat.java (added)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoinFloat.java Tue Nov 24 19:54:19 2009
@@ -0,0 +1,223 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.zebra.pig;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.StringTokenizer;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.pig.TableStorer;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.test.MiniCluster;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Note:
+ *
+ * Make sure you add the build/pig-0.1.0-dev-core.jar to the Classpath of the
+ * app/debug configuration, when run this from inside the Eclipse.
+ *
+ */
+public class TestTableMergeJoinFloat {
+ protected static ExecType execType = ExecType.MAPREDUCE;
+ private static MiniCluster cluster;
+ protected static PigServer pigServer;
+ private static Path pathTable;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ if (System.getProperty("hadoop.log.dir") == null) {
+ String base = new File(".").getPath(); // getAbsolutePath();
+ System
+ .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
+ }
+
+ if (execType == ExecType.MAPREDUCE) {
+ cluster = MiniCluster.buildCluster();
+ pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ } else {
+ pigServer = new PigServer(ExecType.LOCAL);
+ }
+
+ Configuration conf = new Configuration();
+ FileSystem fs = cluster.getFileSystem();
+ Path pathWorking = fs.getWorkingDirectory();
+ pathTable = new Path(pathWorking, "TestTableStorer");
+ System.out.println("pathTable =" + pathTable);
+ BasicTable.Writer writer = new BasicTable.Writer(pathTable,
+ "SF_a:float,SF_b:string,SF_c,SF_d,SF_e,SF_f,SF_g",
+ "[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]", conf);
+ Schema schema = writer.getSchema();
+ System.out.println("typeName" + schema.getColumn("SF_a").getType().pigDataType());
+ Tuple tuple = TypesUtils.createTuple(schema);
+
+ final int numsBatch = 10;
+ final int numsInserters = 1;
+ TableInserter[] inserters = new TableInserter[numsInserters];
+ for (int i = 0; i < numsInserters; i++) {
+ inserters[i] = writer.getInserter("ins" + i, false);
+ }
+
+ for (int b = 0; b < numsBatch; b++) {
+ for (int i = 0; i < numsInserters; i++) {
+ TypesUtils.resetTuple(tuple);
+ for (int k = 0; k < tuple.size(); ++k) {
+ try {
+ if(k==0) {
+ Float f = 32.987f;
+ tuple.set(0, k+b-f);
+ } else {
+ tuple.set(k, b + "_" + i + "" + k);
+ }
+ } catch (ExecException e) {
+ e.printStackTrace();
+ }
+ }
+ inserters[i].insert(new BytesWritable(("key" + i).getBytes()), tuple);
+ }
+ }
+ for (int i = 0; i < numsInserters; i++) {
+ inserters[i].close();
+ }
+ writer.close();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ pigServer.shutdown();
+ }
+
+ /**
+ * Return the name of the routine that called getCurrentMethodName
+ *
+ */
+ public String getCurrentMethodName() {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ (new Throwable()).printStackTrace(pw);
+ pw.flush();
+ String stackTrace = baos.toString();
+ pw.close();
+
+ StringTokenizer tok = new StringTokenizer(stackTrace, "\n");
+ tok.nextToken(); // 'java.lang.Throwable'
+ tok.nextToken(); // 'at ...getCurrentMethodName'
+ String l = tok.nextToken(); // 'at ...<caller to getCurrentRoutine>'
+ // Parse line 3
+ tok = new StringTokenizer(l.trim(), " <(");
+ String t = tok.nextToken(); // 'at'
+ t = tok.nextToken(); // '...<caller to getCurrentRoutine>'
+ return t;
+ }
+
+ @Test
+ public void testStorer() throws ExecException, IOException {
+ /*
+ * Use pig LOAD to load testing data for store
+ */
+ String query = "records = LOAD '" + pathTable.toString()
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query);
+
+
+ String orderby = "srecs = ORDER records BY SF_a;";
+ pigServer.registerQuery(orderby);
+
+ Path newPath = new Path(getCurrentMethodName());
+
+ /*
+ * Table1 creation
+ */
+
+
+ pigServer
+ .store(
+ "srecs",
+ newPath.toString()+"1",
+ TableStorer.class.getCanonicalName()
+ + "('[SF_a, SF_b, SF_c]; [SF_e]')");
+
+ String query3 = "records1 = LOAD '"
+ + newPath.toString() + "1"
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader('SF_a, SF_b', 'sorted');";
+ pigServer.registerQuery(query3);
+ /*
+ * Table2 creation
+ */
+
+ pigServer
+ .store(
+ "srecs",
+ newPath.toString()+"2",
+ TableStorer.class.getCanonicalName()
+ + "('[SF_a, SF_b, SF_c]; [SF_e]')");
+
+
+ String query4 = "records2 = LOAD '"
+ + newPath.toString() + "2"
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query4);
+
+ String join = "joinRecords = JOIN records1 BY SF_a, records2 BY SF_a USING \"merge\";";
+ pigServer.registerQuery(join);
+ // check JOIN content
+ Iterator<Tuple> it3 = pigServer.openIterator("joinRecords");
+ int row = 0;
+ Tuple RowValue3 = null;
+ while (it3.hasNext()) {
+ // Last row value
+ RowValue3 = it3.next();
+ Assert.assertEquals(9, RowValue3.size());
+ row++;
+ if (row == 10) {
+ Assert.assertEquals("9_01", RowValue3.get(1));
+ Assert.assertEquals(-23.987f, RowValue3.get(0));
+ Assert.assertEquals("9_06", RowValue3.get(8));
+ Assert.assertEquals("9_05", RowValue3.get(7));
+ Assert.assertEquals("9_04", RowValue3.get(6));
+ Assert.assertEquals("9_03", RowValue3.get(5));
+ Assert.assertEquals("9_02", RowValue3.get(4));
+ Assert.assertEquals("9_01", RowValue3.get(3));
+ Assert.assertEquals(-23.987f, RowValue3.get(2));
+ }
+ }
+ Assert.assertEquals(10, row);
+ }
+}
Added: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoinInteger.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoinInteger.java?rev=883836&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoinInteger.java (added)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoinInteger.java Tue Nov 24 19:54:19 2009
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.zebra.pig;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.StringTokenizer;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.pig.TableStorer;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.test.MiniCluster;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Note:
+ *
+ * Make sure you add the build/pig-0.1.0-dev-core.jar to the Classpath of the
+ * app/debug configuration, when run this from inside the Eclipse.
+ *
+ */
+public class TestTableMergeJoinInteger {
+ protected static ExecType execType = ExecType.MAPREDUCE;
+ private static MiniCluster cluster;
+ protected static PigServer pigServer;
+ private static Path pathTable;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ if (System.getProperty("hadoop.log.dir") == null) {
+ String base = new File(".").getPath(); // getAbsolutePath();
+ System
+ .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
+ }
+
+ if (execType == ExecType.MAPREDUCE) {
+ cluster = MiniCluster.buildCluster();
+ pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ } else {
+ pigServer = new PigServer(ExecType.LOCAL);
+ }
+
+ Configuration conf = new Configuration();
+ FileSystem fs = cluster.getFileSystem();
+ Path pathWorking = fs.getWorkingDirectory();
+ pathTable = new Path(pathWorking, "TestTableStorer");
+ System.out.println("pathTable =" + pathTable);
+ BasicTable.Writer writer = new BasicTable.Writer(pathTable,
+ "SF_a:int,SF_b:string,SF_c,SF_d,SF_e,SF_f,SF_g",
+ "[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]", conf);
+ Schema schema = writer.getSchema();
+ System.out.println("typeName" + schema.getColumn("SF_a").getType().pigDataType());
+ Tuple tuple = TypesUtils.createTuple(schema);
+
+ final int numsBatch = 10;
+ final int numsInserters = 1;
+ TableInserter[] inserters = new TableInserter[numsInserters];
+ for (int i = 0; i < numsInserters; i++) {
+ inserters[i] = writer.getInserter("ins" + i, false);
+ }
+
+ for (int b = 0; b < numsBatch; b++) {
+ for (int i = 0; i < numsInserters; i++) {
+ TypesUtils.resetTuple(tuple);
+ for (int k = 0; k < tuple.size(); ++k) {
+ try {
+ if(k==0) {
+ tuple.set(0, k+b);
+ } else {
+ tuple.set(k, b + "_" + i + "" + k);
+ }
+ } catch (ExecException e) {
+ e.printStackTrace();
+ }
+ }
+ inserters[i].insert(new BytesWritable(("key" + i).getBytes()), tuple);
+ }
+ }
+ for (int i = 0; i < numsInserters; i++) {
+ inserters[i].close();
+ }
+ writer.close();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ pigServer.shutdown();
+ }
+
+ /**
+ * Return the name of the routine that called getCurrentMethodName
+ *
+ */
+ public String getCurrentMethodName() {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ (new Throwable()).printStackTrace(pw);
+ pw.flush();
+ String stackTrace = baos.toString();
+ pw.close();
+
+ StringTokenizer tok = new StringTokenizer(stackTrace, "\n");
+ tok.nextToken(); // 'java.lang.Throwable'
+ tok.nextToken(); // 'at ...getCurrentMethodName'
+ String l = tok.nextToken(); // 'at ...<caller to getCurrentRoutine>'
+ // Parse line 3
+ tok = new StringTokenizer(l.trim(), " <(");
+ String t = tok.nextToken(); // 'at'
+ t = tok.nextToken(); // '...<caller to getCurrentRoutine>'
+ return t;
+ }
+
+ @Test
+ public void testStorer() throws ExecException, IOException {
+ /*
+ * Use pig LOAD to load testing data for store
+ */
+ String query = "records = LOAD '" + pathTable.toString()
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query);
+
+
+ String orderby = "srecs = ORDER records BY SF_a;";
+ pigServer.registerQuery(orderby);
+
+ Path newPath = new Path(getCurrentMethodName());
+
+ /*
+ * Table1 creation
+ */
+
+
+ pigServer
+ .store(
+ "srecs",
+ newPath.toString()+"1",
+ TableStorer.class.getCanonicalName()
+ + "('[SF_a, SF_b, SF_c]; [SF_e]')");
+
+ String query3 = "records1 = LOAD '"
+ + newPath.toString() + "1"
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader('SF_a, SF_b', 'sorted');";
+ pigServer.registerQuery(query3);
+ /*
+ * Table2 creation
+ */
+
+ pigServer
+ .store(
+ "srecs",
+ newPath.toString()+"2",
+ TableStorer.class.getCanonicalName()
+ + "('[SF_a, SF_b, SF_c]; [SF_e]')");
+
+
+ String query4 = "records2 = LOAD '"
+ + newPath.toString() + "2"
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query4);
+
+ String join = "joinRecords = JOIN records1 BY SF_a, records2 BY SF_a USING \"merge\";";
+ pigServer.registerQuery(join);
+ // check JOIN content
+ Iterator<Tuple> it3 = pigServer.openIterator("joinRecords");
+ int row = 0;
+ Tuple RowValue3 = null;
+ while (it3.hasNext()) {
+ // Last row value
+ RowValue3 = it3.next();
+ Assert.assertEquals(9, RowValue3.size());
+ row++;
+ if (row == 10) {
+ Assert.assertEquals("9_01", RowValue3.get(1));
+ Assert.assertEquals(9, RowValue3.get(0));
+ Assert.assertEquals("9_06", RowValue3.get(8));
+ Assert.assertEquals("9_05", RowValue3.get(7));
+ Assert.assertEquals("9_04", RowValue3.get(6));
+ Assert.assertEquals("9_03", RowValue3.get(5));
+ Assert.assertEquals("9_02", RowValue3.get(4));
+ Assert.assertEquals("9_01", RowValue3.get(3));
+ Assert.assertEquals(9, RowValue3.get(2));
+ }
+ }
+ Assert.assertEquals(10, row);
+ }
+}
Added: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoinMultipleColsSort.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoinMultipleColsSort.java?rev=883836&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoinMultipleColsSort.java (added)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableMergeJoinMultipleColsSort.java Tue Nov 24 19:54:19 2009
@@ -0,0 +1,224 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.zebra.pig;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.StringTokenizer;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.pig.TableStorer;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.test.MiniCluster;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Note:
+ *
+ * Make sure you add the build/pig-0.1.0-dev-core.jar to the Classpath of the
+ * app/debug configuration, when run this from inside the Eclipse.
+ *
+ */
+public class TestTableMergeJoinMultipleColsSort {
+ protected static ExecType execType = ExecType.MAPREDUCE;
+ private static MiniCluster cluster;
+ protected static PigServer pigServer;
+ private static Path pathTable;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ if (System.getProperty("hadoop.log.dir") == null) {
+ String base = new File(".").getPath(); // getAbsolutePath();
+ System
+ .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
+ }
+
+ if (execType == ExecType.MAPREDUCE) {
+ cluster = MiniCluster.buildCluster();
+ pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ } else {
+ pigServer = new PigServer(ExecType.LOCAL);
+ }
+
+ Configuration conf = new Configuration();
+ FileSystem fs = cluster.getFileSystem();
+ Path pathWorking = fs.getWorkingDirectory();
+ pathTable = new Path(pathWorking, "TestTableStorer");
+ System.out.println("pathTable =" + pathTable);
+ BasicTable.Writer writer = new BasicTable.Writer(pathTable,
+ "SF_a:int,SF_b:int,SF_c,SF_d,SF_e,SF_f,SF_g",
+ "[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]", conf);
+ Schema schema = writer.getSchema();
+ System.out.println("typeName" + schema.getColumn("SF_a").getType().pigDataType());
+ Tuple tuple = TypesUtils.createTuple(schema);
+
+ final int numsBatch = 10;
+ final int numsInserters = 1;
+ TableInserter[] inserters = new TableInserter[numsInserters];
+ for (int i = 0; i < numsInserters; i++) {
+ inserters[i] = writer.getInserter("ins" + i, false);
+ }
+
+ for (int b = 0; b < numsBatch; b++) {
+ for (int i = 0; i < numsInserters; i++) {
+ TypesUtils.resetTuple(tuple);
+ for (int k = 0; k < tuple.size(); ++k) {
+ try {
+ if(k==0) {
+ tuple.set(0, k+b);
+ } else if(k==1) {
+ tuple.set(1, numsBatch - b);
+ } else {
+ tuple.set(k, b + "_" + i + "" + k);
+ }
+ } catch (ExecException e) {
+ e.printStackTrace();
+ }
+ }
+ inserters[i].insert(new BytesWritable(("key" + i).getBytes()), tuple);
+ }
+ }
+ for (int i = 0; i < numsInserters; i++) {
+ inserters[i].close();
+ }
+ writer.close();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ pigServer.shutdown();
+ }
+
+ /**
+ * Return the name of the routine that called getCurrentMethodName
+ *
+ */
+ public String getCurrentMethodName() {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ (new Throwable()).printStackTrace(pw);
+ pw.flush();
+ String stackTrace = baos.toString();
+ pw.close();
+
+ StringTokenizer tok = new StringTokenizer(stackTrace, "\n");
+ tok.nextToken(); // 'java.lang.Throwable'
+ tok.nextToken(); // 'at ...getCurrentMethodName'
+ String l = tok.nextToken(); // 'at ...<caller to getCurrentRoutine>'
+ // Parse line 3
+ tok = new StringTokenizer(l.trim(), " <(");
+ String t = tok.nextToken(); // 'at'
+ t = tok.nextToken(); // '...<caller to getCurrentRoutine>'
+ return t;
+ }
+
+ @Test
+ public void testStorer() throws ExecException, IOException {
+ /*
+ * Use pig LOAD to load testing data for store
+ */
+ String query = "records = LOAD '" + pathTable.toString()
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query);
+
+
+ String orderby = "srecs = ORDER records BY SF_a,SF_b;";
+ pigServer.registerQuery(orderby);
+
+ Path newPath = new Path(getCurrentMethodName());
+
+ /*
+ * Table1 creation
+ */
+
+
+ pigServer
+ .store(
+ "srecs",
+ newPath.toString()+"1",
+ TableStorer.class.getCanonicalName()
+ + "('[SF_a, SF_b, SF_c]; [SF_e]')");
+
+ String query3 = "records1 = LOAD '"
+ + newPath.toString() + "1"
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader('SF_a, SF_b', 'sorted');";
+ pigServer.registerQuery(query3);
+ /*
+ * Table2 creation
+ */
+
+ pigServer
+ .store(
+ "srecs",
+ newPath.toString()+"2",
+ TableStorer.class.getCanonicalName()
+ + "('[SF_a, SF_b, SF_c]; [SF_e]')");
+
+
+ String query4 = "records2 = LOAD '"
+ + newPath.toString() + "2"
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query4);
+
+ String join = "joinRecords = JOIN records1 BY (SF_a, SF_b), records2 BY (SF_a,SF_b) USING \"merge\";";
+ pigServer.registerQuery(join);
+ // check JOIN content
+ Iterator<Tuple> it3 = pigServer.openIterator("joinRecords");
+ int row = 0;
+ Tuple RowValue3 = null;
+ while (it3.hasNext()) {
+ // Last row value
+ RowValue3 = it3.next();
+ Assert.assertEquals(9, RowValue3.size());
+ row++;
+ if (row == 10) {
+ Assert.assertEquals(1, RowValue3.get(1));
+ Assert.assertEquals(9, RowValue3.get(0));
+ Assert.assertEquals("9_06", RowValue3.get(8));
+ Assert.assertEquals("9_05", RowValue3.get(7));
+ Assert.assertEquals("9_04", RowValue3.get(6));
+ Assert.assertEquals("9_03", RowValue3.get(5));
+ Assert.assertEquals("9_02", RowValue3.get(4));
+ Assert.assertEquals(1, RowValue3.get(3));
+ Assert.assertEquals(9, RowValue3.get(2));
+ }
+ }
+ Assert.assertEquals(10, row);
+ }
+}
Added: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableSortStorer.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableSortStorer.java?rev=883836&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableSortStorer.java (added)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableSortStorer.java Tue Nov 24 19:54:19 2009
@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.zebra.pig;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Iterator;
+import java.util.StringTokenizer;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.pig.TableStorer;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.test.MiniCluster;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Note:
+ *
+ * Make sure you add the build/pig-0.1.0-dev-core.jar to the Classpath of the
+ * app/debug configuration, when run this from inside the Eclipse.
+ *
+ */
+public class TestTableSortStorer {
+ protected static ExecType execType = ExecType.MAPREDUCE;
+ private static MiniCluster cluster;
+ protected static PigServer pigServer;
+ private static Path pathTable;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ if (System.getProperty("hadoop.log.dir") == null) {
+ String base = new File(".").getPath(); // getAbsolutePath();
+ System
+ .setProperty("hadoop.log.dir", new Path(base).toString() + "./logs");
+ }
+
+ if (execType == ExecType.MAPREDUCE) {
+ cluster = MiniCluster.buildCluster();
+ pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ } else {
+ pigServer = new PigServer(ExecType.LOCAL);
+ }
+
+ Configuration conf = new Configuration();
+ FileSystem fs = cluster.getFileSystem();
+ Path pathWorking = fs.getWorkingDirectory();
+ pathTable = new Path(pathWorking, "TestTableStorer");
+ System.out.println("pathTable =" + pathTable);
+ BasicTable.Writer writer = new BasicTable.Writer(pathTable,
+ "SF_a:string,SF_b,SF_c,SF_d,SF_e,SF_f,SF_g",
+ "[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]", conf);
+ Schema schema = writer.getSchema();
+ Tuple tuple = TypesUtils.createTuple(schema);
+
+ final int numsBatch = 10;
+ final int numsInserters = 1;
+ TableInserter[] inserters = new TableInserter[numsInserters];
+ for (int i = 0; i < numsInserters; i++) {
+ inserters[i] = writer.getInserter("ins" + i, false);
+ }
+
+ for (int b = 0; b < numsBatch; b++) {
+ for (int i = 0; i < numsInserters; i++) {
+ TypesUtils.resetTuple(tuple);
+ for (int k = 0; k < tuple.size(); ++k) {
+ try {
+ tuple.set(k, (9-b) + "_" + i + "" + k);
+ } catch (ExecException e) {
+ e.printStackTrace();
+ }
+ }
+ inserters[i].insert(new BytesWritable(("key" + i).getBytes()), tuple);
+ }
+ }
+ for (int i = 0; i < numsInserters; i++) {
+ inserters[i].close();
+ }
+ writer.close();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ pigServer.shutdown();
+ }
+
+ /**
+ * Return the name of the routine that called getCurrentMethodName
+ *
+ */
+ public String getCurrentMethodName() {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ (new Throwable()).printStackTrace(pw);
+ pw.flush();
+ String stackTrace = baos.toString();
+ pw.close();
+
+ StringTokenizer tok = new StringTokenizer(stackTrace, "\n");
+ tok.nextToken(); // 'java.lang.Throwable'
+ tok.nextToken(); // 'at ...getCurrentMethodName'
+ String l = tok.nextToken(); // 'at ...<caller to getCurrentRoutine>'
+ // Parse line 3
+ tok = new StringTokenizer(l.trim(), " <(");
+ String t = tok.nextToken(); // 'at'
+ t = tok.nextToken(); // '...<caller to getCurrentRoutine>'
+ return t;
+ }
+
+ @Test
+ public void testStorer() throws ExecException, IOException {
+ /*
+ * Use pig LOAD to load testing data for store
+ */
+ String query = "records = LOAD '" + pathTable.toString()
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader();";
+ pigServer.registerQuery(query);
+
+ Iterator<Tuple> it2 = pigServer.openIterator("records");
+ int row0 = 0;
+ Tuple RowValue2 = null;
+ while (it2.hasNext()) {
+ // Last row value
+ RowValue2 = it2.next();
+ row0++;
+ if (row0 == 10) {
+ Assert.assertEquals("0_01", RowValue2.get(1));
+ Assert.assertEquals("0_00", RowValue2.get(0).toString());
+ }
+ }
+ Assert.assertEquals(10, row0);
+
+ String orderby = "srecs = ORDER records BY SF_a;";
+ pigServer.registerQuery(orderby);
+
+ /*
+ * Use pig STORE to store testing data BasicTable.Writer writer = new
+ * BasicTable.Writer(pathTable, "SF_a,SF_b,SF_c,SF_d,SF_e,SF_f,SF_g",
+ * "[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]", false, conf);
+ */
+ Path newPath = new Path(getCurrentMethodName());
+
+ pigServer
+ .store(
+ "srecs",
+ newPath.toString(),
+ TableStorer.class.getCanonicalName()
+ + "('[SF_a, SF_b, SF_c]; [SF_e]')");
+
+ // check new table content
+ String query3 = "newRecords = LOAD '"
+ + newPath.toString()
+ + "' USING org.apache.hadoop.zebra.pig.TableLoader('SF_a, SF_b');";
+ pigServer.registerQuery(query3);
+
+ Iterator<Tuple> it3 = pigServer.openIterator("newRecords");
+ int row = 0;
+ Tuple RowValue3 = null;
+ while (it3.hasNext()) {
+ // Last row value
+ RowValue3 = it3.next();
+ row++;
+ if (row == 10) {
+ Assert.assertEquals("9_01", RowValue3.get(1));
+ Assert.assertEquals("9_00", RowValue3.get(0).toString());
+ }
+ }
+ Assert.assertEquals(10, row);
+ }
+}
Modified: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableStorer.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableStorer.java?rev=883836&r1=883835&r2=883836&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableStorer.java (original)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableStorer.java Tue Nov 24 19:54:19 2009
@@ -22,6 +22,7 @@
import java.io.IOException;
import java.util.Iterator;
+import junit.framework.Assert;
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
@@ -36,6 +37,7 @@
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.executionengine.ExecJob;
import org.apache.pig.data.Tuple;
import org.apache.pig.test.MiniCluster;
import org.junit.After;
@@ -79,7 +81,7 @@
System.out.println("pathTable =" + pathTable);
BasicTable.Writer writer = new BasicTable.Writer(pathTable,
"SF_a,SF_b,SF_c,SF_d,SF_e,SF_f,SF_g",
- "[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]", false, conf);
+ "[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]", conf);
Schema schema = writer.getSchema();
Tuple tuple = TypesUtils.createTuple(schema);
@@ -134,12 +136,13 @@
* BasicTable.Writer(pathTable, "SF_a,SF_b,SF_c,SF_d,SF_e,SF_f,SF_g",
* "[SF_a, SF_b, SF_c]; [SF_e, SF_f, SF_g]", false, conf);
*/
- pigServer
+ ExecJob pigJob = pigServer
.store(
"records",
new Path(pathTable, "store").toString(),
TableStorer.class.getCanonicalName()
+ "('[SF_a, SF_b, SF_c]; [SF_e]')");
+ Assert.assertNull(pigJob.getException());
}
}
\ No newline at end of file
Modified: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestUnionMixedTypes.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestUnionMixedTypes.java?rev=883836&r1=883835&r2=883836&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestUnionMixedTypes.java (original)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestUnionMixedTypes.java Tue Nov 24 19:54:19 2009
@@ -96,7 +96,7 @@
System.out.println("pathTable1 =" + pathTable1);
BasicTable.Writer writer = new BasicTable.Writer(pathTable1, STR_SCHEMA1,
- STR_STORAGE1, false, conf);
+ STR_STORAGE1, conf);
Schema schema = writer.getSchema();
Tuple tuple = TypesUtils.createTuple(schema);
@@ -184,7 +184,7 @@
System.out.println("pathTable2 =" + pathTable2);
BasicTable.Writer writer2 = new BasicTable.Writer(pathTable2, STR_SCHEMA2,
- STR_STORAGE2, false, conf);
+ STR_STORAGE2, conf);
Schema schema2 = writer.getSchema();
Tuple tuple2 = TypesUtils.createTuple(schema2);
@@ -309,16 +309,15 @@
if (j == 1) {
System.out.println("j is : " + j);
- Assert.assertEquals(j + "." + j, cur2.get(0));
- Assert.assertEquals(j + "." + j + j, cur2.get(1));
+ /* The order of t1 and t2 in the union result can vary across runs. */
+ Assert.assertTrue("1.1".equals(cur2.get(0)) || "3.3".equals(cur2.get(0)));
+ Assert.assertTrue("1.11".equals(cur2.get(1)) || "3.33".equals(cur2.get(1)));
}
if (j == 2) {
System.out.println("j is : " + j);
- Assert.assertEquals((j - 1) + "." + (j - 1) + (j - 1) + (j - 1), cur2
- .get(0));
- Assert.assertEquals((j - 1) + "." + (j - 1) + (j - 1) + (j - 1)
- + (j - 1), cur2.get(1));
+ Assert.assertTrue("1.111".equals(cur2.get(0)) || "3.333".equals(cur2.get(0)));
+ Assert.assertTrue("1.1111".equals(cur2.get(1)) || "3.3333".equals(cur2.get(1)));
}
TypesUtils.resetTuple(cur2);
@@ -326,37 +325,37 @@
}// inner while
if (i == 1) {
System.out.println("i is : " + i);
-
- Assert.assertEquals("k11", ((Map) cur.get(1)).get("k1"));
+ Assert.assertTrue("k11".equals(((Map) cur.get(1)).get("k1")) || "k13".equals(((Map) cur.get(1)).get("k1")));
Assert.assertEquals(null, ((Map) cur.get(1)).get("k2"));
- Assert.assertEquals("1", cur.get(2));
+ Assert.assertTrue("1".equals(cur.get(2)) || "3".equals(cur.get(2)));
}
if (i == 2) {
System.out.println("i should see this line. ");
- Assert.assertEquals("k12", ((Map) cur.get(1)).get("k1"));
- Assert.assertEquals("k22", ((Map) cur.get(1)).get("k2"));
- Assert.assertEquals("2", cur.get(2));
+ Assert.assertTrue("k12".equals(((Map) cur.get(1)).get("k1")) || "k14".equals(((Map) cur.get(1)).get("k1")));
+ Assert.assertTrue("k22".equals(((Map) cur.get(1)).get("k2")) || "k24".equals(((Map) cur.get(1)).get("k2")));
+ Assert.assertTrue("2".equals(cur.get(2)) || "4".equals(cur.get(2)));
}
if (i == 3) {
System.out.println("i is : " + i);
- Assert.assertEquals("k13", ((Map) cur.get(1)).get("k1"));
+ Assert.assertTrue("k11".equals(((Map) cur.get(1)).get("k1")) || "k13".equals(((Map) cur.get(1)).get("k1")));
Assert.assertEquals(null, ((Map) cur.get(1)).get("k2"));
- Assert.assertEquals("3", cur.get(2));
+ Assert.assertTrue("1".equals(cur.get(2)) || "3".equals(cur.get(2)));
}
if (i == 4) {
System.out.println("i should see this line. ");
- Assert.assertEquals("k14", ((Map) cur.get(1)).get("k1"));
- Assert.assertEquals("k24", ((Map) cur.get(1)).get("k2"));
- Assert.assertEquals("4", cur.get(2));
+ Assert.assertTrue("k12".equals(((Map) cur.get(1)).get("k1")) || "k14".equals(((Map) cur.get(1)).get("k1")));
+ Assert.assertTrue("k22".equals(((Map) cur.get(1)).get("k2")) || "k24".equals(((Map) cur.get(1)).get("k2")));
+ Assert.assertTrue("2".equals(cur.get(2)) || "4".equals(cur.get(2)));
}
}// outer while
Assert.assertEquals(4, i);
}
+
@Test
// one common field only
public void testReader2() throws ExecException, IOException {
@@ -399,7 +398,7 @@
Assert.assertEquals(null, ((Map) cur.get(0)).get("k2"));
try {
cur.get(1);
- Assert.fail("should throw index out of bound excepiotn");
+ Assert.fail("should throw index out of bound exception");
} catch (Exception e) {
e.printStackTrace();
}
@@ -411,7 +410,7 @@
Assert.assertEquals(null, ((Map) cur.get(0)).get("k2"));
try {
cur.get(1);
- Assert.fail("should throw index out of bound excepiotn");
+ Assert.fail("should throw index out of bound exception");
} catch (Exception e) {
e.printStackTrace();
}
@@ -423,7 +422,7 @@
Assert.assertEquals(null, ((Map) cur.get(0)).get("k2"));
try {
cur.get(1);
- Assert.fail("should throw index out of bound excepiotn");
+ Assert.fail("should throw index out of bound exception");
} catch (Exception e) {
e.printStackTrace();
}
@@ -455,34 +454,35 @@
i++;
System.out.println(" line : " + i + " : " + cur.toString());
+
+
if (i == 1) {
System.out.println("i is : " + i);
- Assert.assertEquals("world1", cur.get(0));
+ Assert.assertTrue("world1".equals(cur.get(0)) || cur.get(0) == null);
try {
cur.get(1);
- Assert.fail("should throw index out of bound excepiotn");
+ Assert.fail("should throw index out of bound exception");
} catch (Exception e) {
e.printStackTrace();
}
}
if (i == 2) {
-
- Assert.assertEquals("world2", cur.get(0));
+ Assert.assertTrue("world2".equals(cur.get(0)) || cur.get(0) == null);
try {
cur.get(1);
- Assert.fail("should throw index out of bound excepiotn");
+ Assert.fail("should throw index out of bound exception");
} catch (Exception e) {
e.printStackTrace();
}
}
if (i == 3) {
- Assert.assertEquals(null, cur.get(0));
+ Assert.assertTrue("world1".equals(cur.get(0)) || cur.get(0) == null);
try {
cur.get(1);
- Assert.fail("should throw index out of bound excepiotn");
+ Assert.fail("should throw index out of bound exception");
} catch (Exception e) {
e.printStackTrace();
}
@@ -490,10 +490,10 @@
if (i == 4) {
- Assert.assertEquals(null, cur.get(0));
+ Assert.assertTrue("world2".equals(cur.get(0)) || cur.get(0) == null);
try {
cur.get(1);
- Assert.fail("should throw index out of bound excepiotn");
+ Assert.fail("should throw index out of bound exception");
} catch (Exception e) {
e.printStackTrace();
}
Added: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/tfile/KVGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/tfile/KVGenerator.java?rev=883836&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/tfile/KVGenerator.java (added)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/tfile/KVGenerator.java Tue Nov 24 19:54:19 2009
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.zebra.tfile;
+
+import java.util.Random;
+
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.zebra.tfile.RandomDistribution.DiscreteRNG;
+
+/**
+ * Generate random <key, value> pairs.
+ */
+class KVGenerator {
+ private final Random random;
+ private final byte[][] dict;
+ private final boolean sorted;
+ private final DiscreteRNG keyLenRNG, valLenRNG;
+ private BytesWritable lastKey;
+ private static final int MIN_KEY_LEN = 4;
+ private final byte prefix[] = new byte[MIN_KEY_LEN];
+
+ public KVGenerator(Random random, boolean sorted, DiscreteRNG keyLenRNG,
+ DiscreteRNG valLenRNG, DiscreteRNG wordLenRNG, int dictSize) {
+ this.random = random;
+ dict = new byte[dictSize][];
+ this.sorted = sorted;
+ this.keyLenRNG = keyLenRNG;
+ this.valLenRNG = valLenRNG;
+ for (int i = 0; i < dictSize; ++i) {
+ int wordLen = wordLenRNG.nextInt();
+ dict[i] = new byte[wordLen];
+ random.nextBytes(dict[i]);
+ }
+ lastKey = new BytesWritable();
+ fillKey(lastKey);
+ }
+
+ private void fillKey(BytesWritable o) {
+ int len = keyLenRNG.nextInt();
+ if (len < MIN_KEY_LEN) len = MIN_KEY_LEN;
+ o.setSize(len);
+ int n = MIN_KEY_LEN;
+ while (n < len) {
+ byte[] word = dict[random.nextInt(dict.length)];
+ int l = Math.min(word.length, len - n);
+ System.arraycopy(word, 0, o.get(), n, l);
+ n += l;
+ }
+ if (sorted
+ && WritableComparator.compareBytes(lastKey.get(), MIN_KEY_LEN, lastKey
+ .getSize()
+ - MIN_KEY_LEN, o.get(), MIN_KEY_LEN, o.getSize() - MIN_KEY_LEN) > 0) {
+ incrementPrefix();
+ }
+
+ System.arraycopy(prefix, 0, o.get(), 0, MIN_KEY_LEN);
+ lastKey.set(o);
+ }
+
+ private void fillValue(BytesWritable o) {
+ int len = valLenRNG.nextInt();
+ o.setSize(len);
+ int n = 0;
+ while (n < len) {
+ byte[] word = dict[random.nextInt(dict.length)];
+ int l = Math.min(word.length, len - n);
+ System.arraycopy(word, 0, o.get(), n, l);
+ n += l;
+ }
+ }
+
+ private void incrementPrefix() {
+ for (int i = MIN_KEY_LEN - 1; i >= 0; --i) {
+ ++prefix[i];
+ if (prefix[i] != 0) return;
+ }
+
+ throw new RuntimeException("Prefix overflown");
+ }
+
+ public void next(BytesWritable key, BytesWritable value, boolean dupKey) {
+ if (dupKey) {
+ key.set(lastKey);
+ }
+ else {
+ fillKey(key);
+ }
+ fillValue(value);
+ }
+}
Added: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/tfile/KeySampler.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/tfile/KeySampler.java?rev=883836&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/tfile/KeySampler.java (added)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/tfile/KeySampler.java Tue Nov 24 19:54:19 2009
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.zebra.tfile;
+
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.tfile.RandomDistribution.DiscreteRNG;
+
+class KeySampler {
+ Random random;
+ int min, max;
+ DiscreteRNG keyLenRNG;
+ private static final int MIN_KEY_LEN = 4;
+
+ public KeySampler(Random random, RawComparable first, RawComparable last,
+ DiscreteRNG keyLenRNG) throws IOException {
+ this.random = random;
+ min = keyPrefixToInt(first);
+ max = keyPrefixToInt(last);
+ this.keyLenRNG = keyLenRNG;
+ }
+
+ private int keyPrefixToInt(RawComparable key) throws IOException {
+ byte[] b = key.buffer();
+ int o = key.offset();
+ return (b[o] & 0xff) << 24 | (b[o + 1] & 0xff) << 16
+ | (b[o + 2] & 0xff) << 8 | (b[o + 3] & 0xff);
+ }
+
+ public void next(BytesWritable key) {
+ key.setSize(Math.max(MIN_KEY_LEN, keyLenRNG.nextInt()));
+ random.nextBytes(key.get());
+ int n = random.nextInt(max - min) + min;
+ byte[] b = key.get();
+ b[0] = (byte) (n >> 24);
+ b[1] = (byte) (n >> 16);
+ b[2] = (byte) (n >> 8);
+ b[3] = (byte) n;
+ }
+}
Added: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/tfile/NanoTimer.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/tfile/NanoTimer.java?rev=883836&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/tfile/NanoTimer.java (added)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/tfile/NanoTimer.java Tue Nov 24 19:54:19 2009
@@ -0,0 +1,193 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.zebra.tfile;
+
+/**
+ * A nano-second timer.
+ */
+public class NanoTimer {
+ private long last = -1;
+ private boolean started = false;
+ private long cumulate = 0;
+
+ /**
+ * Constructor
+ *
+ * @param start
+ * Start the timer upon construction.
+ */
+ public NanoTimer(boolean start) {
+ if (start) this.start();
+ }
+
+ /**
+ * Start the timer.
+ *
+ * Note: No effect if timer is already started.
+ */
+ public void start() {
+ if (!this.started) {
+ this.last = System.nanoTime();
+ this.started = true;
+ }
+ }
+
+ /**
+ * Stop the timer.
+ *
+ * Note: No effect if timer is already stopped.
+ */
+ public void stop() {
+ if (this.started) {
+ this.started = false;
+ this.cumulate += System.nanoTime() - this.last;
+ }
+ }
+
+ /**
+ * Read the timer.
+ *
+ * @return the elapsed time in nano-seconds. Note: If the timer is never
+ * started before, -1 is returned.
+ */
+ public long read() {
+ if (!readable()) return -1;
+
+ return this.cumulate;
+ }
+
+ /**
+ * Reset the timer.
+ */
+ public void reset() {
+ this.last = -1;
+ this.started = false;
+ this.cumulate = 0;
+ }
+
+ /**
+ * Checking whether the timer is started
+ *
+ * @return true if timer is started.
+ */
+ public boolean isStarted() {
+ return this.started;
+ }
+
+ /**
+ * Format the elapsed time to a human understandable string.
+ *
+ * Note: If timer is never started, "ERR" will be returned.
+ */
+ public String toString() {
+ if (!readable()) {
+ return "ERR";
+ }
+
+ return NanoTimer.nanoTimeToString(this.cumulate);
+ }
+
+ /**
+ * A utility method to format a time duration in nano seconds into a human
+ * understandable stirng.
+ *
+ * @param t
+ * Time duration in nano seconds.
+ * @return String representation.
+ */
+ public static String nanoTimeToString(long t) {
+ if (t < 0) return "ERR";
+
+ if (t == 0) return "0";
+
+ if (t < 1000) {
+ return t + "ns";
+ }
+
+ double us = (double) t / 1000;
+ if (us < 1000) {
+ return String.format("%.2fus", us);
+ }
+
+ double ms = us / 1000;
+ if (ms < 1000) {
+ return String.format("%.2fms", ms);
+ }
+
+ double ss = ms / 1000;
+ if (ss < 1000) {
+ return String.format("%.2fs", ss);
+ }
+
+ long mm = (long) ss / 60;
+ ss -= mm * 60;
+ long hh = mm / 60;
+ mm -= hh * 60;
+ long dd = hh / 24;
+ hh -= dd * 24;
+
+ if (dd > 0) {
+ return String.format("%dd%dh", dd, hh);
+ }
+
+ if (hh > 0) {
+ return String.format("%dh%dm", hh, mm);
+ }
+
+ if (mm > 0) {
+ return String.format("%dm%.1fs", mm, ss);
+ }
+
+ return String.format("%.2fs", ss);
+
+ /**
+ * StringBuilder sb = new StringBuilder(); String sep = "";
+ *
+ * if (dd > 0) { String unit = (dd > 1) ? "days" : "day";
+ * sb.append(String.format("%s%d%s", sep, dd, unit)); sep = " "; }
+ *
+ * if (hh > 0) { String unit = (hh > 1) ? "hrs" : "hr";
+ * sb.append(String.format("%s%d%s", sep, hh, unit)); sep = " "; }
+ *
+ * if (mm > 0) { String unit = (mm > 1) ? "mins" : "min";
+ * sb.append(String.format("%s%d%s", sep, mm, unit)); sep = " "; }
+ *
+ * if (ss > 0) { String unit = (ss > 1) ? "secs" : "sec";
+ * sb.append(String.format("%s%.3f%s", sep, ss, unit)); sep = " "; }
+ *
+ * return sb.toString();
+ */
+ }
+
+ private boolean readable() {
+ return this.last != -1;
+ }
+
+ /**
+ * Simple tester.
+ *
+ * @param args
+ */
+ public static void main(String[] args) {
+ long i = 7;
+
+ for (int x = 0; x < 20; ++x, i *= 7) {
+ System.out.println(NanoTimer.nanoTimeToString(i));
+ }
+ }
+}
+
Added: hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/tfile/RandomDistribution.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/tfile/RandomDistribution.java?rev=883836&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/tfile/RandomDistribution.java (added)
+++ hadoop/pig/branches/load-store-redesign/contrib/zebra/src/test/org/apache/hadoop/zebra/tfile/RandomDistribution.java Tue Nov 24 19:54:19 2009
@@ -0,0 +1,266 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.zebra.tfile;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Random;
+
+/**
+ * A class that generates random numbers that follow some distribution.
+ */
+public class RandomDistribution {
+ /**
+ * Interface for discrete (integer) random distributions.
+ */
+ public static interface DiscreteRNG {
+ /**
+ * Get the next random number
+ *
+ * @return the next random number.
+ */
+ public int nextInt();
+ }
+
+ /**
+ * P(i)=1/(max-min)
+ */
+ public static final class Flat implements DiscreteRNG {
+ private final Random random;
+ private final int min;
+ private final int max;
+
+ /**
+ * Generate random integers from min (inclusive) to max (exclusive)
+ * following even distribution.
+ *
+ * @param random
+ * The basic random number generator.
+ * @param min
+ * Minimum integer
+ * @param max
+ * maximum integer (exclusive).
+ *
+ */
+ public Flat(Random random, int min, int max) {
+ if (min >= max) {
+ throw new IllegalArgumentException("Invalid range");
+ }
+ this.random = random;
+ this.min = min;
+ this.max = max;
+ }
+
+ /**
+ * @see DiscreteRNG#nextInt()
+ */
+ @Override
+ public int nextInt() {
+ return random.nextInt(max - min) + min;
+ }
+ }
+
+ /**
+ * Zipf distribution. The ratio of the probabilities of integer i and j is
+ * defined as follows:
+ *
+ * P(i)/P(j)=((j-min+1)/(i-min+1))^sigma.
+ */
+ public static final class Zipf implements DiscreteRNG {
+ private static final double DEFAULT_EPSILON = 0.001;
+ private final Random random;
+ private final ArrayList<Integer> k;
+ private final ArrayList<Double> v;
+
+ /**
+ * Constructor
+ *
+ * @param r
+ * The random number generator.
+ * @param min
+ * minimum integer (inclusvie)
+ * @param max
+ * maximum integer (exclusive)
+ * @param sigma
+ * parameter sigma. (sigma > 1.0)
+ */
+ public Zipf(Random r, int min, int max, double sigma) {
+ this(r, min, max, sigma, DEFAULT_EPSILON);
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param r
+ * The random number generator.
+ * @param min
+ * minimum integer (inclusvie)
+ * @param max
+ * maximum integer (exclusive)
+ * @param sigma
+ * parameter sigma. (sigma > 1.0)
+ * @param epsilon
+ * Allowable error percentage (0 < epsilon < 1.0).
+ */
+ public Zipf(Random r, int min, int max, double sigma, double epsilon) {
+ if ((max <= min) || (sigma <= 1) || (epsilon <= 0)
+ || (epsilon >= 0.5)) {
+ throw new IllegalArgumentException("Invalid arguments");
+ }
+ random = r;
+ k = new ArrayList<Integer>();
+ v = new ArrayList<Double>();
+
+ double sum = 0;
+ int last = -1;
+ for (int i = min; i < max; ++i) {
+ sum += Math.exp(-sigma * Math.log(i - min + 1));
+ if ((last == -1) || i * (1 - epsilon) > last) {
+ k.add(i);
+ v.add(sum);
+ last = i;
+ }
+ }
+
+ if (last != max - 1) {
+ k.add(max - 1);
+ v.add(sum);
+ }
+
+ v.set(v.size() - 1, 1.0);
+
+ for (int i = v.size() - 2; i >= 0; --i) {
+ v.set(i, v.get(i) / sum);
+ }
+ }
+
+ /**
+ * @see DiscreteRNG#nextInt()
+ */
+ @Override
+ public int nextInt() {
+ double d = random.nextDouble();
+ int idx = Collections.binarySearch(v, d);
+
+ if (idx > 0) {
+ ++idx;
+ }
+ else {
+ idx = -(idx + 1);
+ }
+
+ if (idx >= v.size()) {
+ idx = v.size() - 1;
+ }
+
+ if (idx == 0) {
+ return k.get(0);
+ }
+
+ int ceiling = k.get(idx);
+ int lower = k.get(idx - 1);
+
+ return ceiling - random.nextInt(ceiling - lower);
+ }
+ }
+
+ /**
+ * Binomial distribution.
+ *
+ * P(k)=select(n, k)*p^k*(1-p)^(n-k) (k = 0, 1, ..., n)
+ *
+ * P(k)=select(max-min-1, k-min)*p^(k-min)*(1-p)^(k-min)*(1-p)^(max-k-1)
+ */
+ public static final class Binomial implements DiscreteRNG {
+ private final Random random;
+ private final int min;
+ private final int n;
+ private final double[] v;
+
+ private static double select(int n, int k) {
+ double ret = 1.0;
+ for (int i = k + 1; i <= n; ++i) {
+ ret *= (double) i / (i - k);
+ }
+ return ret;
+ }
+
+ private static double power(double p, int k) {
+ return Math.exp(k * Math.log(p));
+ }
+
+ /**
+ * Generate random integers from min (inclusive) to max (exclusive)
+ * following Binomial distribution.
+ *
+ * @param random
+ * The basic random number generator.
+ * @param min
+ * Minimum integer
+ * @param max
+ * maximum integer (exclusive).
+ * @param p
+ * parameter.
+ *
+ */
+ public Binomial(Random random, int min, int max, double p) {
+ if (min >= max) {
+ throw new IllegalArgumentException("Invalid range");
+ }
+ this.random = random;
+ this.min = min;
+ this.n = max - min - 1;
+ if (n > 0) {
+ v = new double[n + 1];
+ double sum = 0.0;
+ for (int i = 0; i <= n; ++i) {
+ sum += select(n, i) * power(p, i) * power(1 - p, n - i);
+ v[i] = sum;
+ }
+ for (int i = 0; i <= n; ++i) {
+ v[i] /= sum;
+ }
+ }
+ else {
+ v = null;
+ }
+ }
+
+ /**
+ * @see DiscreteRNG#nextInt()
+ */
+ @Override
+ public int nextInt() {
+ if (v == null) {
+ return min;
+ }
+ double d = random.nextDouble();
+ int idx = Arrays.binarySearch(v, d);
+ if (idx > 0) {
+ ++idx;
+ } else {
+ idx = -(idx + 1);
+ }
+
+ if (idx >= v.length) {
+ idx = v.length - 1;
+ }
+ return idx + min;
+ }
+ }
+}