You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ra...@apache.org on 2009/10/12 06:29:14 UTC
svn commit: r824221 - in
/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra: io/ types/
Author: rangadi
Date: Mon Oct 12 04:29:13 2009
New Revision: 824221
URL: http://svn.apache.org/viewvc?rev=824221&view=rev
Log:
PIG-986. Second commit. forgot to svn-add new files in the previous commit.
Added:
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName1.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName2.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName3.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName4.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName5.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName6.java
hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnGroupName.java
Added: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName1.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName1.java?rev=824221&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName1.java (added)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName1.java Mon Oct 12 04:29:13 2009
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.zebra.io;
+
+
+import java.io.IOException;
+import java.util.List;
+import junit.framework.Assert;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.io.TableScanner;
+import org.apache.hadoop.zebra.io.BasicTable.Reader.RangeSplit;
+import org.apache.hadoop.zebra.types.ParseException;
+import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.Tuple;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ *
+ * Test projections on complicated column types.
+ *
+ */
+public class TestColumnGroupName1 {
+
+ final static String STR_SCHEMA = "s1:bool, s2:int, s3:long, s4:float, s5:string, s6:bytes";
+ final static String STR_STORAGE = "[s1, s2] as PI; [s3, s4] as General; [s5, s6] as ULT";
+ private static Configuration conf;
+ private static Path path;
+ private static FileSystem fs;
+
+ @BeforeClass
+ public static void setUpOnce() throws IOException {
+
+ conf = new Configuration();
+ conf.setInt("table.output.tfile.minBlock.size", 64 * 1024);
+ conf.setInt("table.input.split.minSize", 64 * 1024);
+ conf.set("table.output.tfile.compression", "none");
+
+ RawLocalFileSystem rawLFS = new RawLocalFileSystem();
+ fs = new LocalFileSystem(rawLFS);
+ path = new Path(fs.getWorkingDirectory(), "TestSimple");
+ fs = path.getFileSystem(conf);
+ // drop any previous tables
+ BasicTable.drop(path, conf);
+ BasicTable.Writer writer = new BasicTable.Writer(path, STR_SCHEMA,
+ STR_STORAGE, false, conf);
+ writer.finish();
+ Schema schema = writer.getSchema();
+ Tuple tuple = TypesUtils.createTuple(schema);
+ BasicTable.Writer writer1 = new BasicTable.Writer(path, conf);
+ int part = 0;
+ TableInserter inserter = writer1.getInserter("part" + part, true);
+ TypesUtils.resetTuple(tuple);
+
+ // insert data in row 1
+ int row = 0;
+ tuple.set(0, true); // bool
+ tuple.set(1, 1); // int
+ tuple.set(2, 1001L); // long
+ tuple.set(3, 1.1); // float
+ tuple.set(4, "hello world 1"); // string
+ tuple.set(5, new DataByteArray("hello byte 1")); // byte
+
+ inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
+ .getBytes()), tuple);
+
+ // insert data in row 2
+ row++;
+ tuple.set(0, false);
+ tuple.set(1, 2); // int
+ tuple.set(2, 1002L); // long
+ tuple.set(3, 3.1); // float
+ tuple.set(4, "hello world 2"); // string
+ tuple.set(5, new DataByteArray("hello byte 2")); // byte
+ inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
+ .getBytes()), tuple);
+
+ // finish building table, closing out the inserter, writer, writer1
+ inserter.close();
+ writer1.finish();
+ writer.close();
+ }
+
+ @AfterClass
+ public static void tearDownOnce() throws IOException {
+ BasicTable.drop(path, conf);
+ }
+
+ // Test simple projection
+ @Test
+ public void testReadSimple1() throws IOException, ParseException {
+ String projection = new String("s6,s5,s4,s3,s2,s1");
+ BasicTable.Reader reader = new BasicTable.Reader(path, conf);
+ reader.setProjection(projection);
+ List<RangeSplit> splits = reader.rangeSplit(1);
+ TableScanner scanner = reader.getScanner(splits.get(0), true);
+ BytesWritable key = new BytesWritable();
+ Tuple RowValue = TypesUtils.createTuple(scanner.getSchema());
+
+ scanner.getKey(key);
+ Assert.assertEquals(key, new BytesWritable("k11".getBytes()));
+ scanner.getValue(RowValue);
+ Assert.assertEquals(true, RowValue.get(5));
+ Assert.assertEquals(1, RowValue.get(4));
+ Assert.assertEquals(1001L, RowValue.get(3));
+ Assert.assertEquals(1.1, RowValue.get(2));
+ Assert.assertEquals("hello world 1", RowValue.get(1));
+ Assert.assertEquals("hello byte 1", RowValue.get(0).toString());
+ scanner.advance();
+ scanner.getKey(key);
+ Assert.assertEquals(key, new BytesWritable("k12".getBytes()));
+ scanner.getValue(RowValue);
+ Assert.assertEquals(false, RowValue.get(5));
+ Assert.assertEquals(2, RowValue.get(4));
+ Assert.assertEquals(1002L, RowValue.get(3));
+ Assert.assertEquals(3.1, RowValue.get(2));
+ Assert.assertEquals("hello world 2", RowValue.get(1));
+ Assert.assertEquals("hello byte 2", RowValue.get(0).toString());
+ }
+
+ // test stitch,
+ @Test
+ public void testReadSimpleStitch() throws IOException, ParseException {
+ String projection2 = new String("s5, s1");
+ BasicTable.Reader reader = new BasicTable.Reader(path, conf);
+ reader.setProjection(projection2);
+ List<RangeSplit> splits = reader.rangeSplit(1);
+ TableScanner scanner = reader.getScanner(splits.get(0), true);
+ BytesWritable key = new BytesWritable();
+ Tuple RowValue = TypesUtils.createTuple(scanner.getSchema());
+ scanner.getKey(key);
+ Assert.assertEquals(key, new BytesWritable("k11".getBytes()));
+ scanner.getValue(RowValue);
+ Assert.assertEquals("hello world 1", RowValue.get(0));
+ Assert.assertEquals(true, RowValue.get(1));
+
+ scanner.advance();
+
+ scanner.getKey(key);
+ Assert.assertEquals(key, new BytesWritable("k12".getBytes()));
+ scanner.getValue(RowValue);
+ Assert.assertEquals("hello world 2", RowValue.get(0));
+ Assert.assertEquals(false, RowValue.get(1));
+
+ reader.close();
+ }
+}
\ No newline at end of file
Added: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName2.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName2.java?rev=824221&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName2.java (added)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName2.java Mon Oct 12 04:29:13 2009
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.zebra.io;
+
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.Tuple;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ *
+ * Test projections on complicated column types.
+ *
+ */
+public class TestColumnGroupName2 {
+
+ final static String STR_SCHEMA = "s1:bool, s2:int, s3:long, s4:float, s5:string, s6:bytes";
+ final static String STR_STORAGE = "[s1, s2] as PI secure by user:chaow perm:777; [s3, s4] as General; [s5, s6] as ULT";
+ private static Configuration conf;
+ private static Path path;
+ private static FileSystem fs;
+
+ @BeforeClass
+ public static void setUpOnce() throws IOException {
+
+ conf = new Configuration();
+ conf.setInt("table.output.tfile.minBlock.size", 64 * 1024);
+ conf.setInt("table.input.split.minSize", 64 * 1024);
+ conf.set("table.output.tfile.compression", "none");
+
+ RawLocalFileSystem rawLFS = new RawLocalFileSystem();
+ fs = new LocalFileSystem(rawLFS);
+ path = new Path(fs.getWorkingDirectory(), "TestColumnGroupName");
+ fs = path.getFileSystem(conf);
+ // drop any previous tables
+ BasicTable.drop(path, conf);
+ BasicTable.Writer writer = new BasicTable.Writer(path, STR_SCHEMA,
+ STR_STORAGE, false, conf);
+ writer.finish();
+ Schema schema = writer.getSchema();
+ Tuple tuple = TypesUtils.createTuple(schema);
+ BasicTable.Writer writer1 = new BasicTable.Writer(path, conf);
+ int part = 0;
+ TableInserter inserter = writer1.getInserter("part" + part, true);
+ TypesUtils.resetTuple(tuple);
+
+ // insert data in row 1
+ int row = 0;
+ tuple.set(0, true); // bool
+ tuple.set(1, 1); // int
+ tuple.set(2, 1001L); // long
+ tuple.set(3, 1.1); // float
+ tuple.set(4, "hello world 1"); // string
+ tuple.set(5, new DataByteArray("hello byte 1")); // byte
+
+ inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
+ .getBytes()), tuple);
+
+ // insert data in row 2
+ row++;
+ tuple.set(0, false);
+ tuple.set(1, 2); // int
+ tuple.set(2, 1002L); // long
+ tuple.set(3, 3.1); // float
+ tuple.set(4, "hello world 2"); // string
+ tuple.set(5, new DataByteArray("hello byte 2")); // byte
+ inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
+ .getBytes()), tuple);
+
+ // finish building table, closing out the inserter, writer, writer1
+ inserter.close();
+ writer1.finish();
+ writer.close();
+ }
+
+ @AfterClass
+ public static void tearDownOnce() throws IOException {
+ // BasicTable.drop(path, conf);
+ }
+
+ // Test simple projection
+ @Test
+ public void test1() throws IOException, Exception {
+ BasicTable.dumpInfo(path.toString(), System.out, conf);
+ }
+}
\ No newline at end of file
Added: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName3.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName3.java?rev=824221&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName3.java (added)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName3.java Mon Oct 12 04:29:13 2009
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.zebra.io;
+
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.Tuple;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ *
+ * Test projections on complicated column types.
+ *
+ */
+public class TestColumnGroupName3 {
+
+ final static String STR_SCHEMA = "s1:bool, s2:int, s3:long, s4:float, s5:string, s6:bytes";
+ final static String STR_STORAGE = "[s1, s2]; [s3, s4]; [s5, s6]";
+ private static Configuration conf;
+ private static Path path;
+ private static FileSystem fs;
+
+ @BeforeClass
+ public static void setUpOnce() throws IOException {
+
+ conf = new Configuration();
+ conf.setInt("table.output.tfile.minBlock.size", 64 * 1024);
+ conf.setInt("table.input.split.minSize", 64 * 1024);
+ conf.set("table.output.tfile.compression", "none");
+
+ RawLocalFileSystem rawLFS = new RawLocalFileSystem();
+ fs = new LocalFileSystem(rawLFS);
+ path = new Path(fs.getWorkingDirectory(), "TestColumnGroupName");
+ fs = path.getFileSystem(conf);
+ // drop any previous tables
+ BasicTable.drop(path, conf);
+ BasicTable.Writer writer = new BasicTable.Writer(path, STR_SCHEMA,
+ STR_STORAGE, false, conf);
+ writer.finish();
+ Schema schema = writer.getSchema();
+ Tuple tuple = TypesUtils.createTuple(schema);
+ BasicTable.Writer writer1 = new BasicTable.Writer(path, conf);
+ int part = 0;
+ TableInserter inserter = writer1.getInserter("part" + part, true);
+ TypesUtils.resetTuple(tuple);
+
+ // insert data in row 1
+ int row = 0;
+ tuple.set(0, true); // bool
+ tuple.set(1, 1); // int
+ tuple.set(2, 1001L); // long
+ tuple.set(3, 1.1); // float
+ tuple.set(4, "hello world 1"); // string
+ tuple.set(5, new DataByteArray("hello byte 1")); // byte
+
+ inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
+ .getBytes()), tuple);
+
+ // insert data in row 2
+ row++;
+ tuple.set(0, false);
+ tuple.set(1, 2); // int
+ tuple.set(2, 1002L); // long
+ tuple.set(3, 3.1); // float
+ tuple.set(4, "hello world 2"); // string
+ tuple.set(5, new DataByteArray("hello byte 2")); // byte
+ inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
+ .getBytes()), tuple);
+
+ // finish building table, closing out the inserter, writer, writer1
+ inserter.close();
+ writer1.finish();
+ writer.close();
+ }
+
+ @AfterClass
+ public static void tearDownOnce() throws IOException {
+ BasicTable.drop(path, conf);
+ }
+
+ // Test simple projection
+ @Test
+ public void test1() throws IOException, Exception {
+ BasicTable.dumpInfo(path.toString(), System.out, conf);
+ }
+}
\ No newline at end of file
Added: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName4.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName4.java?rev=824221&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName4.java (added)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName4.java Mon Oct 12 04:29:13 2009
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.zebra.io;
+
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.Tuple;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ *
+ * Test projections on complicated column types.
+ *
+ */
+public class TestColumnGroupName4 {
+
+ final static String STR_SCHEMA = "s1:bool, s2:int, s3:long, s4:float, s5:string, s6:bytes";
+ final static String STR_STORAGE = "[s1, s2] as CG1; [s3, s4]";
+ private static Configuration conf;
+ private static Path path;
+ private static FileSystem fs;
+
+ @BeforeClass
+ public static void setUpOnce() throws IOException {
+
+ conf = new Configuration();
+ conf.setInt("table.output.tfile.minBlock.size", 64 * 1024);
+ conf.setInt("table.input.split.minSize", 64 * 1024);
+ conf.set("table.output.tfile.compression", "none");
+
+ RawLocalFileSystem rawLFS = new RawLocalFileSystem();
+ fs = new LocalFileSystem(rawLFS);
+ path = new Path(fs.getWorkingDirectory(), "TestColumnGroupName");
+ fs = path.getFileSystem(conf);
+ // drop any previous tables
+ BasicTable.drop(path, conf);
+ BasicTable.Writer writer = new BasicTable.Writer(path, STR_SCHEMA,
+ STR_STORAGE, false, conf);
+ writer.finish();
+ Schema schema = writer.getSchema();
+ Tuple tuple = TypesUtils.createTuple(schema);
+ BasicTable.Writer writer1 = new BasicTable.Writer(path, conf);
+ int part = 0;
+ TableInserter inserter = writer1.getInserter("part" + part, true);
+ TypesUtils.resetTuple(tuple);
+
+ // insert data in row 1
+ int row = 0;
+ tuple.set(0, true); // bool
+ tuple.set(1, 1); // int
+ tuple.set(2, 1001L); // long
+ tuple.set(3, 1.1); // float
+ tuple.set(4, "hello world 1"); // string
+ tuple.set(5, new DataByteArray("hello byte 1")); // byte
+
+ inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
+ .getBytes()), tuple);
+
+ // insert data in row 2
+ row++;
+ tuple.set(0, false);
+ tuple.set(1, 2); // int
+ tuple.set(2, 1002L); // long
+ tuple.set(3, 3.1); // float
+ tuple.set(4, "hello world 2"); // string
+ tuple.set(5, new DataByteArray("hello byte 2")); // byte
+ inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
+ .getBytes()), tuple);
+
+ // finish building table, closing out the inserter, writer, writer1
+ inserter.close();
+ writer1.finish();
+ writer.close();
+ }
+
+ @AfterClass
+ public static void tearDownOnce() throws IOException {
+ BasicTable.drop(path, conf);
+ }
+
+ // Test simple projection
+ @Test
+ public void test1() throws IOException, Exception {
+ BasicTable.dumpInfo(path.toString(), System.out, conf);
+ }
+}
\ No newline at end of file
Added: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName5.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName5.java?rev=824221&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName5.java (added)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName5.java Mon Oct 12 04:29:13 2009
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.zebra.io;
+
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.Tuple;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ *
+ * Test projections on complicated column types.
+ *
+ */
+public class TestColumnGroupName5 {
+
+ final static String STR_SCHEMA = "s1:bool, s2:int, s3:long, s4:float, s5:string, s6:bytes, s7:int, s8: int, s9:int, s10:int, s11:int, s12:int";
+ final static String STR_STORAGE = "[s1]; [s2]; [s3]; [s4]; [s5]; [s6]; [s7]; [s8]; [s9]; [s10]; [s11]; [s12]";
+ private static Configuration conf;
+ private static Path path;
+ private static FileSystem fs;
+
+ @BeforeClass
+ public static void setUpOnce() throws IOException {
+
+ conf = new Configuration();
+ conf.setInt("table.output.tfile.minBlock.size", 64 * 1024);
+ conf.setInt("table.input.split.minSize", 64 * 1024);
+ conf.set("table.output.tfile.compression", "none");
+
+ RawLocalFileSystem rawLFS = new RawLocalFileSystem();
+ fs = new LocalFileSystem(rawLFS);
+ path = new Path(fs.getWorkingDirectory(), "TestColumnGroupName");
+ fs = path.getFileSystem(conf);
+ // drop any previous tables
+ BasicTable.drop(path, conf);
+ BasicTable.Writer writer = new BasicTable.Writer(path, STR_SCHEMA,
+ STR_STORAGE, false, conf);
+ writer.finish();
+ Schema schema = writer.getSchema();
+ Tuple tuple = TypesUtils.createTuple(schema);
+ BasicTable.Writer writer1 = new BasicTable.Writer(path, conf);
+ int part = 0;
+ TableInserter inserter = writer1.getInserter("part" + part, true);
+ TypesUtils.resetTuple(tuple);
+
+ // insert data in row 1
+ int row = 0;
+ tuple.set(0, true); // bool
+ tuple.set(1, 1); // int
+ tuple.set(2, 1001L); // long
+ tuple.set(3, 1.1); // float
+ tuple.set(4, "hello world 1"); // string
+ tuple.set(5, new DataByteArray("hello byte 1")); // byte
+ tuple.set(6, 1); // int
+ tuple.set(7, 1); // int
+ tuple.set(8, 1); // int
+ tuple.set(9, 1); // int
+ tuple.set(10, 1); // int
+ tuple.set(11, 1); // int
+
+ inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
+ .getBytes()), tuple);
+
+ // insert data in row 2
+ row++;
+ tuple.set(0, false);
+ tuple.set(1, 2); // int
+ tuple.set(2, 1002L); // long
+ tuple.set(3, 3.1); // float
+ tuple.set(4, "hello world 2"); // string
+ tuple.set(5, new DataByteArray("hello byte 2")); // byte
+ tuple.set(6, 2); // int
+ tuple.set(7, 2); // int
+ tuple.set(8, 2); // int
+ tuple.set(9, 2); // int
+ tuple.set(10, 2); // int
+ tuple.set(11, 2); // int
+
+ inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
+ .getBytes()), tuple);
+
+ // finish building table, closing out the inserter, writer, writer1
+ inserter.close();
+ writer1.finish();
+ writer.close();
+ }
+
+ @AfterClass
+ public static void tearDownOnce() throws IOException {
+ BasicTable.drop(path, conf);
+ }
+
+ // Test simple projection
+ @Test
+ public void test1() throws IOException, Exception {
+ BasicTable.dumpInfo(path.toString(), System.out, conf);
+ }
+}
Added: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName6.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName6.java?rev=824221&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName6.java (added)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupName6.java Mon Oct 12 04:29:13 2009
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.zebra.io;
+
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.Tuple;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ *
+ * Test projections on complicated column types.
+ *
+ */
+public class TestColumnGroupName6 {
+
+ final static String STR_SCHEMA = "s1:bool, s2:int, s3:long, s4:float, s5:string, s6:bytes, s7:int, s8: int, s9:int, s10:int, s11:int, s12:int";
+ final static String STR_STORAGE = "[s1] as CG0; [s2] as CG1; [s3] as CG2; [s4] as CG3; [s5] as CG4; [s6] as CG5; [s7] as CG6; [s8] as CG7; [s9] as CG8; [s10] as CG9; [s11] as CG10; [s12]";
+ private static Configuration conf;
+ private static Path path;
+ private static FileSystem fs;
+
+ @BeforeClass
+ public static void setUpOnce() throws IOException {
+
+ conf = new Configuration();
+ conf.setInt("table.output.tfile.minBlock.size", 64 * 1024);
+ conf.setInt("table.input.split.minSize", 64 * 1024);
+ conf.set("table.output.tfile.compression", "none");
+
+ RawLocalFileSystem rawLFS = new RawLocalFileSystem();
+ fs = new LocalFileSystem(rawLFS);
+ path = new Path(fs.getWorkingDirectory(), "TestColumnGroupName");
+ fs = path.getFileSystem(conf);
+ // drop any previous tables
+ BasicTable.drop(path, conf);
+ BasicTable.Writer writer = new BasicTable.Writer(path, STR_SCHEMA,
+ STR_STORAGE, false, conf);
+ writer.finish();
+ Schema schema = writer.getSchema();
+ Tuple tuple = TypesUtils.createTuple(schema);
+ BasicTable.Writer writer1 = new BasicTable.Writer(path, conf);
+ int part = 0;
+ TableInserter inserter = writer1.getInserter("part" + part, true);
+ TypesUtils.resetTuple(tuple);
+
+ // insert data in row 1
+ int row = 0;
+ tuple.set(0, true); // bool
+ tuple.set(1, 1); // int
+ tuple.set(2, 1001L); // long
+ tuple.set(3, 1.1); // float
+ tuple.set(4, "hello world 1"); // string
+ tuple.set(5, new DataByteArray("hello byte 1")); // byte
+ tuple.set(6, 1); // int
+ tuple.set(7, 1); // int
+ tuple.set(8, 1); // int
+ tuple.set(9, 1); // int
+ tuple.set(10, 1); // int
+ tuple.set(11, 1); // int
+
+ inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
+ .getBytes()), tuple);
+
+ // insert data in row 2
+ row++;
+ tuple.set(0, false);
+ tuple.set(1, 2); // int
+ tuple.set(2, 1002L); // long
+ tuple.set(3, 3.1); // float
+ tuple.set(4, "hello world 2"); // string
+ tuple.set(5, new DataByteArray("hello byte 2")); // byte
+ tuple.set(6, 2); // int
+ tuple.set(7, 2); // int
+ tuple.set(8, 2); // int
+ tuple.set(9, 2); // int
+ tuple.set(10, 2); // int
+ tuple.set(11, 2); // int
+
+ inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
+ .getBytes()), tuple);
+
+ // finish building table, closing out the inserter, writer, writer1
+ inserter.close();
+ writer1.finish();
+ writer.close();
+ }
+
+ @AfterClass
+ public static void tearDownOnce() throws IOException {
+ BasicTable.drop(path, conf);
+ }
+
+ // Test simple projection
+ @Test
+ public void test1() throws IOException, Exception {
+ BasicTable.dumpInfo(path.toString(), System.out, conf);
+ }
+}
Added: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnGroupName.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnGroupName.java?rev=824221&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnGroupName.java (added)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestColumnGroupName.java Mon Oct 12 04:29:13 2009
@@ -0,0 +1,369 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.zebra.types;
+
+import java.io.StringReader;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.HashSet;
+import junit.framework.Assert;
+
+import org.apache.hadoop.zebra.types.CGSchema;
+import org.apache.hadoop.zebra.types.ColumnType;
+import org.apache.hadoop.zebra.types.ParseException;
+import org.apache.hadoop.zebra.types.Partition;
+import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.types.TableSchemaParser;
+import org.apache.hadoop.zebra.types.Schema.ColumnSchema;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestColumnGroupName {
+ String strSch = "f1:int, f2:long, f3:float, f4:bool, f5:string, f6:bytes";
+ TableSchemaParser parser;
+ Schema schema;
+
+ @Before
+ public void init() throws ParseException {
+ parser = new TableSchemaParser(new StringReader(strSch));
+ schema = parser.RecordSchema(null);
+ }
+
+ @Test
+ public void testSchema() throws ParseException {
+ ColumnSchema f1 = schema.getColumn(0);
+ Assert.assertEquals("f1", f1.name);
+ Assert.assertEquals(ColumnType.INT, f1.type);
+ ColumnSchema f2 = schema.getColumn(1);
+ Assert.assertEquals("f2", f2.name);
+ Assert.assertEquals(ColumnType.LONG, f2.type);
+ ColumnSchema f3 = schema.getColumn(2);
+ Assert.assertEquals("f3", f3.name);
+ Assert.assertEquals(ColumnType.FLOAT, f3.type);
+ ColumnSchema f4 = schema.getColumn(3);
+ Assert.assertEquals("f4", f4.name);
+ Assert.assertEquals(ColumnType.BOOL, f4.type);
+ ColumnSchema f5 = schema.getColumn(4);
+ Assert.assertEquals("f5", f5.name);
+ Assert.assertEquals(ColumnType.STRING, f5.type);
+ ColumnSchema f6 = schema.getColumn(5);
+ Assert.assertEquals("f6", f6.name);
+ Assert.assertEquals(ColumnType.BYTES, f6.type);
+
+ System.out.println(schema.toString());
+ }
+
+ @Test
+ public void testStorageValid1() {
+ try {
+ String strStorage = "[f1, f2] as PI; [f3, f4] as General secure by user:joe perm:640 COMPRESS BY gz SERIALIZE BY avro; [f5, f6] as ULT";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+
+ // 3 column group;
+ int size = cgschemas.length;
+ Assert.assertEquals(size, 3);
+ System.out.println("********** Column Groups **********");
+ for (int i = 0; i < cgschemas.length; i++) {
+ System.out.println(cgschemas[i]);
+ System.out.println("--------------------------------");
+ }
+ CGSchema cgs1 = cgschemas[0];
+ CGSchema cgs2 = cgschemas[1];
+ CGSchema cgs3 = cgschemas[2];
+
+ ColumnSchema f11 = cgs1.getSchema().getColumn(0);
+ Assert.assertEquals("f1", f11.name);
+ Assert.assertEquals(ColumnType.INT, f11.type);
+ ColumnSchema f12 = cgs1.getSchema().getColumn(1);
+ Assert.assertEquals("f2", f12.name);
+ Assert.assertEquals(ColumnType.LONG, f12.type);
+ ColumnSchema f21 = cgs2.getSchema().getColumn(0);
+ Assert.assertEquals("f3", f21.name);
+ Assert.assertEquals(ColumnType.FLOAT, f21.type);
+ ColumnSchema f22 = cgs2.getSchema().getColumn(1);
+ Assert.assertEquals("f4", f22.name);
+ Assert.assertEquals(ColumnType.BOOL, f22.type);
+ ColumnSchema f31 = cgs3.getSchema().getColumn(0);
+ Assert.assertEquals("f5", f31.name);
+ Assert.assertEquals(ColumnType.STRING, f31.type);
+ ColumnSchema f32 = cgs3.getSchema().getColumn(1);
+ Assert.assertEquals("f6", f32.name);
+ Assert.assertEquals(ColumnType.BYTES, f32.type);
+
+ Assert.assertEquals(cgs1.getCompressor(), "gz");
+ Assert.assertEquals(cgs1.getSerializer(), "pig");
+ Assert.assertEquals(cgs2.getCompressor(), "gz");
+ Assert.assertEquals(cgs2.getSerializer(), "avro");
+ Assert.assertEquals(cgs3.getCompressor(), "gz");
+ Assert.assertEquals(cgs3.getSerializer(), "pig");
+
+ //Assert.assertEquals(cgs2.getOwner(), "joe");
+ //Assert.assertEquals(cgs2.getGroup(), "secure");
+ //Assert.assertEquals(cgs2.getPerm(), (short) Short.parseShort("640", 8));
+ Assert.assertEquals(cgs1.getName(), "PI");
+ Assert.assertEquals(cgs2.getName(), "General");
+ Assert.assertEquals(cgs3.getName(), "ULT");
+
+ System.out.println("*********** Column Map **********");
+ Map<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>> colmap = p
+ .getPartitionInfo().getColMap();
+ Assert.assertEquals(colmap.size(), 6);
+ Iterator<Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>>> it = colmap
+ .entrySet().iterator();
+ for (int i = 0; i < colmap.size(); i++) {
+ Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>> entry = (Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>>) it
+ .next();
+ String name = entry.getKey();
+ HashSet<Partition.PartitionInfo.ColumnMappingEntry> hs = entry
+ .getValue();
+ Iterator<Partition.PartitionInfo.ColumnMappingEntry> it1 = hs
+ .iterator();
+ for (int j = 0; j < hs.size(); j++) {
+ Partition.PartitionInfo.ColumnMappingEntry cme = (Partition.PartitionInfo.ColumnMappingEntry) it1
+ .next();
+ System.out.println("[Column = " + name + " CG = " + cme.getCGIndex()
+ + "." + cme.getFieldIndex() + "]");
+ if (i == 0 && j == 0) {
+ Assert.assertEquals(name, "f6");
+ Assert.assertEquals(cme.getCGIndex(), 2);
+ Assert.assertEquals(cme.getFieldIndex(), 1);
+ } else if (i == 1 && j == 0) {
+ Assert.assertEquals(name, "f1");
+ Assert.assertEquals(cme.getCGIndex(), 0);
+ Assert.assertEquals(cme.getFieldIndex(), 0);
+ } else if (i == 2 && j == 0) {
+ Assert.assertEquals(name, "f3");
+ Assert.assertEquals(cme.getCGIndex(), 1);
+ Assert.assertEquals(cme.getFieldIndex(), 0);
+ } else if (i == 3 && j == 0) {
+ Assert.assertEquals(name, "f2");
+ Assert.assertEquals(cme.getCGIndex(), 0);
+ Assert.assertEquals(cme.getFieldIndex(), 1);
+ } else if (i == 4 && j == 0) {
+ Assert.assertEquals(name, "f5");
+ Assert.assertEquals(cme.getCGIndex(), 2);
+ Assert.assertEquals(cme.getFieldIndex(), 0);
+ } else if (i == 5 && j == 0) {
+ Assert.assertEquals(name, "f4");
+ Assert.assertEquals(cme.getCGIndex(), 1);
+ Assert.assertEquals(cme.getFieldIndex(), 1);
+ }
+ }
+ }
+ } catch (Exception e) {
+ Assert.assertTrue(false);
+ }
+ }
+
+ @Test
+ public void testStorageValid2() {
+ try {
+ String strStorage = "[f1, f2] serialize by avro compress by gz; [f3, f4] SERIALIZE BY avro COMPRESS BY gz; [f5, f6]";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+
+ Assert.assertEquals(cgschemas.length, 3);
+ CGSchema cgs1 = cgschemas[0];
+ System.out.println(cgs1);
+ CGSchema cgs2 = cgschemas[1];
+ System.out.println(cgs2);
+ CGSchema cgs3 = cgschemas[2];
+ System.out.println(cgs3);
+
+ Assert.assertEquals(cgs1.getName(), "CG0");
+ Assert.assertEquals(cgs2.getName(), "CG1");
+ Assert.assertEquals(cgs3.getName(), "CG2");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testStorageValid3() {
+ try {
+ String strStorage = "[f1, f2] as PI serialize by avro compress by gz; [f3, f4] as General SERIALIZE BY avro COMPRESS BY gz; [f5, f6]";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+
+ Assert.assertEquals(cgschemas.length, 3);
+ CGSchema cgs1 = cgschemas[0];
+ System.out.println(cgs1);
+ CGSchema cgs2 = cgschemas[1];
+ System.out.println(cgs2);
+ CGSchema cgs3 = cgschemas[2];
+ System.out.println(cgs3);
+
+ Assert.assertEquals(cgs1.getName(), "PI");
+ Assert.assertEquals(cgs2.getName(), "General");
+ Assert.assertEquals(cgs3.getName(), "CG0");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testStorageValid4() {
+ try {
+ String strStorage = "[f1, f2] as C1 serialize by avro compress by gz; [f3, f4] as C2 SERIALIZE BY avro COMPRESS BY gz; as C3";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+
+ Assert.assertEquals(cgschemas.length, 3);
+ CGSchema cgs1 = cgschemas[0];
+ System.out.println(cgs1);
+ CGSchema cgs2 = cgschemas[1];
+ System.out.println(cgs2);
+ CGSchema cgs3 = cgschemas[2];
+ System.out.println(cgs3);
+
+ Assert.assertEquals(cgs1.getName(), "C1");
+ Assert.assertEquals(cgs2.getName(), "C2");
+ Assert.assertEquals(cgs3.getName(), "C3");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testStorageValid5() {
+ try {
+ String strStorage = "[f1, f2] as C1 serialize by avro compress by gz; [f3, f4] as C2 SERIALIZE BY avro COMPRESS BY gz;";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+
+ Assert.assertEquals(cgschemas.length, 3);
+ CGSchema cgs1 = cgschemas[0];
+ System.out.println(cgs1);
+ CGSchema cgs2 = cgschemas[1];
+ System.out.println(cgs2);
+ CGSchema cgs3 = cgschemas[2];
+ System.out.println(cgs3);
+
+ Assert.assertEquals(cgs1.getName(), "C1");
+ Assert.assertEquals(cgs2.getName(), "C2");
+ Assert.assertEquals(cgs3.getName(), "CG0");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testStorageValid6() {
+ try {
+ String strStorage = "[f1, f2] as PI serialize by avro compress by gz; [f3, f4] SERIALIZE BY avro COMPRESS BY gz; [f5, f6] as CG0";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+
+ Assert.assertEquals(cgschemas.length, 3);
+ CGSchema cgs1 = cgschemas[0];
+ System.out.println(cgs1);
+ CGSchema cgs2 = cgschemas[1];
+ System.out.println(cgs2);
+ CGSchema cgs3 = cgschemas[2];
+ System.out.println(cgs3);
+
+ Assert.assertEquals(cgs1.getName(), "PI");
+ Assert.assertEquals(cgs2.getName(), "CG1");
+ Assert.assertEquals(cgs3.getName(), "CG0");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testStorageValid7() {
+ try {
+ String strStorage = "[f1, f2] as PI serialize by avro compress by gz; [f3, f4] as Pi SERIALIZE BY avro COMPRESS BY gz; [f5, f6] as CG100";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+
+ Assert.assertEquals(cgschemas.length, 3);
+ CGSchema cgs1 = cgschemas[0];
+ System.out.println(cgs1);
+ CGSchema cgs2 = cgschemas[1];
+ System.out.println(cgs2);
+ CGSchema cgs3 = cgschemas[2];
+ System.out.println(cgs3);
+
+ Assert.assertEquals(cgs1.getName(), "PI");
+ Assert.assertEquals(cgs2.getName(), "Pi");
+ Assert.assertEquals(cgs3.getName(), "CG100");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ /*
+ * There is no default column group and C3 is skipped.
+ */
+ @Test
+ public void testStorageValid8() {
+ try {
+ String strStorage = "[f1, f2] as C1 serialize by avro compress by gz; [f3, f4, f5, f6] as C2 SERIALIZE BY avro COMPRESS BY gz; as C3 compress by gz";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+ Assert.assertEquals(cgschemas.length, 2);
+ CGSchema cgs1 = cgschemas[0];
+ System.out.println(cgs1);
+ CGSchema cgs2 = cgschemas[1];
+ System.out.println(cgs2);
+
+ Assert.assertEquals(cgs1.getName(), "C1");
+ Assert.assertEquals(cgs2.getName(), "C2");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testStorageInvalid1() {
+ try {
+ String strStorage = "[f1, f2] as C1 serialize by avro compress by gz; [f3, f4] as C1 SERIALIZE BY avro COMPRESS BY gz; [f5, f6] as C3";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+ CGSchema cgs1 = cgschemas[0];
+ System.out.println(cgs1);
+
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ String errMsg = e.getMessage();
+ String str = "Duplicate column group names.";
+ Assert.assertEquals(errMsg, str);
+ }
+ }
+
+ @Test
+ public void testStorageInvalid2() {
+ try {
+ String strStorage = "[f1, f2] serialize by avro compress by gz as C1; [f3, f4] as C2 SERIALIZE BY avro COMPRESS BY gz; [f5, f6] as C3";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+ CGSchema cgs1 = cgschemas[0];
+ System.out.println(cgs1);
+
+ Assert.assertTrue(false);
+ } catch (ParseException e) {
+ System.out.println(e.getMessage());
+ } catch (IOException e) {
+ Assert.assertTrue(false);
+ }
+ }
+}
\ No newline at end of file