You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2009/08/12 00:27:47 UTC
svn commit: r803312 [16/16] - in /hadoop/pig/trunk: ./ contrib/zebra/
contrib/zebra/docs/ contrib/zebra/src/ contrib/zebra/src/java/
contrib/zebra/src/java/org/ contrib/zebra/src/java/org/apache/
contrib/zebra/src/java/org/apache/hadoop/ contrib/zebra/...
Added: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc3.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc3.java?rev=803312&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc3.java (added)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageMisc3.java Tue Aug 11 22:27:44 2009
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.zebra.types;
+
+import java.io.StringReader;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.HashSet;
+import junit.framework.Assert;
+
+import org.apache.hadoop.zebra.types.CGSchema;
+import org.apache.hadoop.zebra.types.ColumnType;
+import org.apache.hadoop.zebra.types.ParseException;
+import org.apache.hadoop.zebra.types.Partition;
+import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.types.TableSchemaParser;
+import org.apache.hadoop.zebra.types.Schema.ColumnSchema;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestStorageMisc3 {
+ String strSch = "c:collection(r:record(f1:int, f2:int)), m1:map(int)";
+ TableSchemaParser parser;
+ Schema schema;
+
+ @Before
+ public void init() throws ParseException {
+ parser = new TableSchemaParser(new StringReader(strSch));
+ schema = parser.RecordSchema(null);
+ }
+
+ @Test
+ public void testStorageValid1() {
+ try {
+ String strStorage = "[c] compress by gzip; [m1] serialize by avro";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+
+ // 2 column group;
+ int size = cgschemas.length;
+ Assert.assertEquals(size, 2);
+ System.out.println("********** Column Groups **********");
+ for (int i = 0; i < cgschemas.length; i++) {
+ System.out.println(cgschemas[i]);
+ System.out.println("--------------------------------");
+ }
+ CGSchema cgs1 = cgschemas[0];
+ CGSchema cgs2 = cgschemas[1];
+
+ ColumnSchema f11 = cgs1.getSchema().getColumn(0);
+ Assert.assertEquals("c", f11.name);
+ Assert.assertEquals(ColumnType.COLLECTION, f11.type);
+
+ ColumnSchema f21 = cgs2.getSchema().getColumn(0);
+ Assert.assertEquals("m1", f21.name);
+ Assert.assertEquals(ColumnType.MAP, f21.type);
+
+ Assert.assertEquals(cgs1.getCompressor(), "gzip");
+ Assert.assertEquals(cgs2.getSerializer(), "avro");
+
+ System.out.println("*********** Column Map **********");
+ Map<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>> colmap = p
+ .getPartitionInfo().getColMap();
+ Assert.assertEquals(colmap.size(), 2);
+ Iterator<Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>>> it = colmap
+ .entrySet().iterator();
+ for (int i = 0; i < colmap.size(); i++) {
+ Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>> entry = (Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>>) it
+ .next();
+ String name = entry.getKey();
+ HashSet<Partition.PartitionInfo.ColumnMappingEntry> hs = entry
+ .getValue();
+ Iterator<Partition.PartitionInfo.ColumnMappingEntry> it1 = hs
+ .iterator();
+ for (int j = 0; j < hs.size(); j++) {
+ Partition.PartitionInfo.ColumnMappingEntry cme = (Partition.PartitionInfo.ColumnMappingEntry) it1
+ .next();
+ System.out.println("[Column = " + name + " CG = " + cme.getCGIndex()
+ + "." + cme.getFieldIndex() + "]");
+ if (i == 0 && j == 0) {
+ Assert.assertEquals(name, "c");
+ Assert.assertEquals(cme.getCGIndex(), 0);
+ Assert.assertEquals(cme.getFieldIndex(), 0);
+ } else if (i == 1 && j == 0) {
+ Assert.assertEquals(name, "m1");
+ Assert.assertEquals(cme.getCGIndex(), 1);
+ Assert.assertEquals(cme.getFieldIndex(), 0);
+ }
+ }
+ }
+ } catch (Exception e) {
+ Assert.assertTrue(false);
+ }
+ }
+
+ @Test
+ public void testStorageValid2() {
+ try {
+ String strStorage = "[c.r] compress by gzip; [m1] serialize by avro";
+ Partition p = new Partition(schema.toString(), strStorage);
+ Assert.assertTrue(false);
+ CGSchema[] cgschemas = p.getCGSchemas();
+
+ // 2 column group;
+ int size = cgschemas.length;
+ Assert.assertEquals(size, 2);
+ System.out.println("********** Column Groups **********");
+ for (int i = 0; i < cgschemas.length; i++) {
+ System.out.println(cgschemas[i]);
+ System.out.println("--------------------------------");
+ }
+ CGSchema cgs1 = cgschemas[0];
+ CGSchema cgs2 = cgschemas[1];
+
+ ColumnSchema f11 = cgs1.getSchema().getColumn(0);
+ Assert.assertEquals("c.r", f11.name);
+ Assert.assertEquals(ColumnType.RECORD, f11.type);
+
+ ColumnSchema f21 = cgs2.getSchema().getColumn(0);
+ Assert.assertEquals("m1", f21.name);
+ Assert.assertEquals(ColumnType.MAP, f21.type);
+
+ Assert.assertEquals(cgs1.getCompressor(), "gzip");
+ Assert.assertEquals(cgs2.getSerializer(), "avro");
+
+ System.out.println("*********** Column Map **********");
+ Map<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>> colmap = p
+ .getPartitionInfo().getColMap();
+ Assert.assertEquals(colmap.size(), 2);
+ Iterator<Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>>> it = colmap
+ .entrySet().iterator();
+ for (int i = 0; i < colmap.size(); i++) {
+ Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>> entry = (Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>>) it
+ .next();
+ String name = entry.getKey();
+ HashSet<Partition.PartitionInfo.ColumnMappingEntry> hs = entry
+ .getValue();
+ Iterator<Partition.PartitionInfo.ColumnMappingEntry> it1 = hs
+ .iterator();
+ for (int j = 0; j < hs.size(); j++) {
+ Partition.PartitionInfo.ColumnMappingEntry cme = (Partition.PartitionInfo.ColumnMappingEntry) it1
+ .next();
+ System.out.println("[Column = " + name + " CG = " + cme.getCGIndex()
+ + "." + cme.getFieldIndex() + "]");
+ if (i == 0 && j == 0) {
+ Assert.assertEquals(name, "m1");
+ Assert.assertEquals(cme.getCGIndex(), 1);
+ Assert.assertEquals(cme.getFieldIndex(), 0);
+ } else if (i == 1 && j == 0) {
+ Assert.assertEquals(name, "c.r");
+ Assert.assertEquals(cme.getCGIndex(), 0);
+ Assert.assertEquals(cme.getFieldIndex(), 0);
+ }
+ }
+ }
+ } catch (Exception e) {
+ }
+ }
+
+ @Test
+ public void testStorageValid3() {
+ try {
+ String strStorage = "[c.r.f1] compress by gzip; [m1] serialize by avro";
+ Partition p = new Partition(schema.toString(), strStorage);
+ Assert.assertTrue(false);
+ CGSchema[] cgschemas = p.getCGSchemas();
+
+ // 3 column group;
+ int size = cgschemas.length;
+ Assert.assertEquals(size, 3);
+ System.out.println("********** Column Groups **********");
+ for (int i = 0; i < cgschemas.length; i++) {
+ System.out.println(cgschemas[i]);
+ System.out.println("--------------------------------");
+ }
+ CGSchema cgs1 = cgschemas[0];
+ CGSchema cgs2 = cgschemas[1];
+ CGSchema cgs3 = cgschemas[2];
+
+ ColumnSchema f11 = cgs1.getSchema().getColumn(0);
+ Assert.assertEquals("c.r.f1", f11.name);
+ Assert.assertEquals(ColumnType.INT, f11.type);
+
+ ColumnSchema f21 = cgs2.getSchema().getColumn(0);
+ Assert.assertEquals("m1", f21.name);
+ Assert.assertEquals(ColumnType.MAP, f21.type);
+
+ ColumnSchema f31 = cgs3.getSchema().getColumn(0);
+ Assert.assertEquals("c.r.f2", f31.name);
+ Assert.assertEquals(ColumnType.INT, f31.type);
+
+ Assert.assertEquals(cgs1.getCompressor(), "gzip");
+ Assert.assertEquals(cgs1.getSerializer(), "pig");
+ Assert.assertEquals(cgs2.getCompressor(), "lzo2");
+ Assert.assertEquals(cgs2.getSerializer(), "avro");
+ Assert.assertEquals(cgs3.getCompressor(), "lzo2");
+ Assert.assertEquals(cgs3.getSerializer(), "pig");
+
+ System.out.println("*********** Column Map **********");
+ Map<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>> colmap = p
+ .getPartitionInfo().getColMap();
+ Assert.assertEquals(colmap.size(), 3);
+ Iterator<Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>>> it = colmap
+ .entrySet().iterator();
+ for (int i = 0; i < colmap.size(); i++) {
+ Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>> entry = (Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>>) it
+ .next();
+ String name = entry.getKey();
+ HashSet<Partition.PartitionInfo.ColumnMappingEntry> hs = entry
+ .getValue();
+ Iterator<Partition.PartitionInfo.ColumnMappingEntry> it1 = hs
+ .iterator();
+ for (int j = 0; j < hs.size(); j++) {
+ Partition.PartitionInfo.ColumnMappingEntry cme = (Partition.PartitionInfo.ColumnMappingEntry) it1
+ .next();
+ System.out.println("[Column = " + name + " CG = " + cme.getCGIndex()
+ + "." + cme.getFieldIndex() + "]");
+ if (i == 0 && j == 0) {
+ Assert.assertEquals(name, "c.r.f1");
+ Assert.assertEquals(cme.getCGIndex(), 0);
+ Assert.assertEquals(cme.getFieldIndex(), 0);
+ } else if (i == 1 && j == 0) {
+ Assert.assertEquals(name, "c.r.f2");
+ Assert.assertEquals(cme.getCGIndex(), 2);
+ Assert.assertEquals(cme.getFieldIndex(), 0);
+ } else if (i == 2 && j == 0) {
+ Assert.assertEquals(name, "m1");
+ Assert.assertEquals(cme.getCGIndex(), 1);
+ Assert.assertEquals(cme.getFieldIndex(), 0);
+ }
+ }
+ }
+ } catch (Exception e) {
+ }
+ }
+}
\ No newline at end of file
Added: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageRecord.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageRecord.java?rev=803312&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageRecord.java (added)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageRecord.java Tue Aug 11 22:27:44 2009
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.zebra.types;
+
+import java.io.StringReader;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.HashSet;
+import junit.framework.Assert;
+
+import org.apache.hadoop.zebra.types.CGSchema;
+import org.apache.hadoop.zebra.types.ColumnType;
+import org.apache.hadoop.zebra.types.ParseException;
+import org.apache.hadoop.zebra.types.Partition;
+import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.types.TableSchemaParser;
+import org.apache.hadoop.zebra.types.Schema.ColumnSchema;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestStorageRecord {
+ String strSch = "r1:record(f1:int, f2:int), r2:record(r3:record(f3:float, f4))";
+ TableSchemaParser parser;
+ Schema schema;
+
+ @Before
+ public void init() throws ParseException {
+ parser = new TableSchemaParser(new StringReader(strSch));
+ schema = parser.RecordSchema(null);
+ }
+
+ @Test
+ public void testStorageValid1() {
+ try {
+ String strStorage = "[r1.f1, r2.r3.f3]; [r1.f2, r2.r3.f4]";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+
+ // 2 column group;
+ int size = cgschemas.length;
+ Assert.assertEquals(size, 2);
+ System.out.println("********** Column Groups **********");
+ for (int i = 0; i < cgschemas.length; i++) {
+ System.out.println(cgschemas[i]);
+ System.out.println("--------------------------------");
+ }
+ CGSchema cgs1 = cgschemas[0];
+ CGSchema cgs2 = cgschemas[1];
+
+ ColumnSchema f11 = cgs1.getSchema().getColumn(0);
+ Assert.assertEquals("r1.f1", f11.name);
+ Assert.assertEquals(ColumnType.INT, f11.type);
+ ColumnSchema f12 = cgs1.getSchema().getColumn(1);
+ Assert.assertEquals("r2.r3.f3", f12.name);
+ Assert.assertEquals(ColumnType.FLOAT, f12.type);
+
+ ColumnSchema f21 = cgs2.getSchema().getColumn(0);
+ Assert.assertEquals("r1.f2", f21.name);
+ Assert.assertEquals(ColumnType.INT, f21.type);
+ ColumnSchema f22 = cgs2.getSchema().getColumn(1);
+ Assert.assertEquals("r2.r3.f4", f22.name);
+ Assert.assertEquals(ColumnType.BYTES, f22.type);
+
+ System.out.println("*********** Column Map **********");
+ Map<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>> colmap = p
+ .getPartitionInfo().getColMap();
+ Assert.assertEquals(colmap.size(), 4);
+ Iterator<Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>>> it = colmap
+ .entrySet().iterator();
+ for (int i = 0; i < colmap.size(); i++) {
+ Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>> entry = (Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>>) it
+ .next();
+ String name = entry.getKey();
+ HashSet<Partition.PartitionInfo.ColumnMappingEntry> hs = entry
+ .getValue();
+ Iterator<Partition.PartitionInfo.ColumnMappingEntry> it1 = hs
+ .iterator();
+ for (int j = 0; j < hs.size(); j++) {
+ Partition.PartitionInfo.ColumnMappingEntry cme = (Partition.PartitionInfo.ColumnMappingEntry) it1
+ .next();
+ System.out.println("[Column = " + name + " CG = " + cme.getCGIndex()
+ + "." + cme.getFieldIndex() + "]");
+ if (i == 0 && j == 0) {
+ Assert.assertEquals(name, "r1.f1");
+ Assert.assertEquals(cme.getCGIndex(), 0);
+ Assert.assertEquals(cme.getFieldIndex(), 0);
+ } else if (i == 1 && j == 0) {
+ Assert.assertEquals(name, "r1.f2");
+ Assert.assertEquals(cme.getCGIndex(), 1);
+ Assert.assertEquals(cme.getFieldIndex(), 0);
+ } else if (i == 2 && j == 0) {
+ Assert.assertEquals(name, "r2.r3.f3");
+ Assert.assertEquals(cme.getCGIndex(), 0);
+ Assert.assertEquals(cme.getFieldIndex(), 1);
+ } else if (i == 3 && j == 0) {
+ Assert.assertEquals(name, "r2.r3.f4");
+ Assert.assertEquals(cme.getCGIndex(), 1);
+ Assert.assertEquals(cme.getFieldIndex(), 1);
+ }
+ }
+ }
+ } catch (Exception e) {
+ Assert.assertTrue(false);
+ }
+ }
+
+ /*
+ * @Test public void testStorageInvalid1() { try { String strStorage =
+ * "m1#k1"; TableStorageParser parser = new TableStorageParser(new
+ * ByteArrayInputStream(strStorage.getBytes("UTF-8")), null, schema);
+ * ArrayList<CGSchema> schemas = parser.StorageSchema(); CGSchema cgs1 =
+ * schemas.get(0); } catch (Exception e) { String errMsg = e.getMessage();
+ * String str = "Encountered \" <IDENTIFIER> \"m1 \"\" at line 1, column 1.";
+ * System.out.println(errMsg); System.out.println(str);
+ * Assert.assertEquals(errMsg.startsWith(str), true); } }
+ */
+}
Added: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorePrimitive.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorePrimitive.java?rev=803312&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorePrimitive.java (added)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorePrimitive.java Tue Aug 11 22:27:44 2009
@@ -0,0 +1,332 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.zebra.types;
+
+import java.io.StringReader;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.HashSet;
+import junit.framework.Assert;
+
+import org.apache.hadoop.zebra.types.CGSchema;
+import org.apache.hadoop.zebra.types.ColumnType;
+import org.apache.hadoop.zebra.types.ParseException;
+import org.apache.hadoop.zebra.types.Partition;
+import org.apache.hadoop.zebra.types.Schema;
+import org.apache.hadoop.zebra.types.TableSchemaParser;
+import org.apache.hadoop.zebra.types.Schema.ColumnSchema;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestStorePrimitive {
+ String strSch = "f1:int, f2:long, f3:float, f4:bool, f5:string, f6:bytes";
+ TableSchemaParser parser;
+ Schema schema;
+
+ @Before
+ public void init() throws ParseException {
+ parser = new TableSchemaParser(new StringReader(strSch));
+ schema = parser.RecordSchema(null);
+ }
+
+ @Test
+ public void testSchema() throws ParseException {
+ ColumnSchema f1 = schema.getColumn(0);
+ Assert.assertEquals("f1", f1.name);
+ Assert.assertEquals(ColumnType.INT, f1.type);
+ ColumnSchema f2 = schema.getColumn(1);
+ Assert.assertEquals("f2", f2.name);
+ Assert.assertEquals(ColumnType.LONG, f2.type);
+ ColumnSchema f3 = schema.getColumn(2);
+ Assert.assertEquals("f3", f3.name);
+ Assert.assertEquals(ColumnType.FLOAT, f3.type);
+ ColumnSchema f4 = schema.getColumn(3);
+ Assert.assertEquals("f4", f4.name);
+ Assert.assertEquals(ColumnType.BOOL, f4.type);
+ ColumnSchema f5 = schema.getColumn(4);
+ Assert.assertEquals("f5", f5.name);
+ Assert.assertEquals(ColumnType.STRING, f5.type);
+ ColumnSchema f6 = schema.getColumn(5);
+ Assert.assertEquals("f6", f6.name);
+ Assert.assertEquals(ColumnType.BYTES, f6.type);
+
+ System.out.println(schema.toString());
+ }
+
+ @Test
+ public void testStorageValid1() {
+ try {
+ String strStorage = "[f1, f2]; [f3, f4] COMPRESS BY gzip SERIALIZE BY avro";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+
+ // 3 column group;
+ int size = cgschemas.length;
+ Assert.assertEquals(size, 3);
+ System.out.println("********** Column Groups **********");
+ for (int i = 0; i < cgschemas.length; i++) {
+ System.out.println(cgschemas[i]);
+ System.out.println("--------------------------------");
+ }
+ CGSchema cgs1 = cgschemas[0];
+ CGSchema cgs2 = cgschemas[1];
+ CGSchema cgs3 = cgschemas[2];
+
+ ColumnSchema f11 = cgs1.getSchema().getColumn(0);
+ Assert.assertEquals("f1", f11.name);
+ Assert.assertEquals(ColumnType.INT, f11.type);
+ ColumnSchema f12 = cgs1.getSchema().getColumn(1);
+ Assert.assertEquals("f2", f12.name);
+ Assert.assertEquals(ColumnType.LONG, f12.type);
+ ColumnSchema f21 = cgs2.getSchema().getColumn(0);
+ Assert.assertEquals("f3", f21.name);
+ Assert.assertEquals(ColumnType.FLOAT, f21.type);
+ ColumnSchema f22 = cgs2.getSchema().getColumn(1);
+ Assert.assertEquals("f4", f22.name);
+ Assert.assertEquals(ColumnType.BOOL, f22.type);
+ ColumnSchema f31 = cgs3.getSchema().getColumn(0);
+ Assert.assertEquals("f5", f31.name);
+ Assert.assertEquals(ColumnType.STRING, f31.type);
+ ColumnSchema f32 = cgs3.getSchema().getColumn(1);
+ Assert.assertEquals("f6", f32.name);
+ Assert.assertEquals(ColumnType.BYTES, f32.type);
+
+ Assert.assertEquals(cgs1.getCompressor(), "lzo2");
+ Assert.assertEquals(cgs1.getSerializer(), "pig");
+ Assert.assertEquals(cgs2.getCompressor(), "gzip");
+ Assert.assertEquals(cgs2.getSerializer(), "avro");
+ Assert.assertEquals(cgs3.getCompressor(), "lzo2");
+ Assert.assertEquals(cgs3.getSerializer(), "pig");
+
+ System.out.println("*********** Column Map **********");
+ Map<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>> colmap = p
+ .getPartitionInfo().getColMap();
+ Assert.assertEquals(colmap.size(), 6);
+ Iterator<Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>>> it = colmap
+ .entrySet().iterator();
+ for (int i = 0; i < colmap.size(); i++) {
+ Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>> entry = (Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>>) it
+ .next();
+ String name = entry.getKey();
+ HashSet<Partition.PartitionInfo.ColumnMappingEntry> hs = entry
+ .getValue();
+ Iterator<Partition.PartitionInfo.ColumnMappingEntry> it1 = hs
+ .iterator();
+ for (int j = 0; j < hs.size(); j++) {
+ Partition.PartitionInfo.ColumnMappingEntry cme = (Partition.PartitionInfo.ColumnMappingEntry) it1
+ .next();
+ System.out.println("[Column = " + name + " CG = " + cme.getCGIndex()
+ + "." + cme.getFieldIndex() + "]");
+ if (i == 0 && j == 0) {
+ Assert.assertEquals(name, "f6");
+ Assert.assertEquals(cme.getCGIndex(), 2);
+ Assert.assertEquals(cme.getFieldIndex(), 1);
+ } else if (i == 1 && j == 0) {
+ Assert.assertEquals(name, "f1");
+ Assert.assertEquals(cme.getCGIndex(), 0);
+ Assert.assertEquals(cme.getFieldIndex(), 0);
+ } else if (i == 2 && j == 0) {
+ Assert.assertEquals(name, "f3");
+ Assert.assertEquals(cme.getCGIndex(), 1);
+ Assert.assertEquals(cme.getFieldIndex(), 0);
+ } else if (i == 3 && j == 0) {
+ Assert.assertEquals(name, "f2");
+ Assert.assertEquals(cme.getCGIndex(), 0);
+ Assert.assertEquals(cme.getFieldIndex(), 1);
+ } else if (i == 4 && j == 0) {
+ Assert.assertEquals(name, "f5");
+ Assert.assertEquals(cme.getCGIndex(), 2);
+ Assert.assertEquals(cme.getFieldIndex(), 0);
+ } else if (i == 5 && j == 0) {
+ Assert.assertEquals(name, "f4");
+ Assert.assertEquals(cme.getCGIndex(), 1);
+ Assert.assertEquals(cme.getFieldIndex(), 1);
+ }
+ }
+ }
+ } catch (Exception e) {
+ Assert.assertTrue(false);
+ }
+ }
+
+ @Test
+ public void testStorageValid2() {
+ try {
+ String strStorage = "[f1, f2] serialize by avro compress by gzip; [f3, f4] SERIALIZE BY avro COMPRESS BY gzip";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+
+ Assert.assertEquals(cgschemas.length, 3);
+ CGSchema cgs1 = cgschemas[0];
+ System.out.println(cgs1);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testStorageValid3() {
+ try {
+ String strStorage = "";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+ Assert.assertEquals(cgschemas.length, 1);
+ CGSchema cgs1 = cgschemas[0];
+ System.out.println(cgs1);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testStorageInvalid1() {
+ try {
+ String strStorage = "f1";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+ CGSchema cgs1 = cgschemas[0];
+ System.out.println(cgs1);
+ } catch (Exception e) {
+ String errMsg = e.getMessage();
+ String str = "Encountered \" <IDENTIFIER> \"f1 \"\" at line 1, column 1.\nWas expecting one of:\n <EOF> \n \"compress by\" ...\n \"serialize by\" ...\n \"[\" ...\n ";
+ System.out.println(errMsg);
+ System.out.println(str);
+ Assert.assertEquals(errMsg, str);
+ }
+ }
+
+ @Test
+ public void testStorageInvalid2() {
+ try {
+ String strStorage = "[f100]";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+ CGSchema cgs1 = cgschemas[0];
+ System.out.println(cgs1);
+ } catch (Exception e) {
+ String errMsg = e.getMessage();
+ String str = "Column f100 not defined in schema";
+ System.out.println(errMsg);
+ System.out.println(str);
+ Assert.assertEquals(errMsg, str);
+ }
+ }
+
+ @Test
+ public void testStorageInvalid3() {
+ try {
+ String strStorage = "f1:long";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+ CGSchema cgs1 = cgschemas[0];
+ System.out.println(cgs1);
+ } catch (Exception e) {
+ String errMsg = e.getMessage();
+ String str = "Encountered \" <IDENTIFIER> \"f1 \"\" at line 1, column 1.\nWas expecting one of:\n <EOF> \n \"compress by\" ...\n \"serialize by\" ...\n \"[\" ...\n ";
+ System.out.println(errMsg);
+ System.out.println(str);
+ Assert.assertEquals(errMsg, str);
+ }
+ }
+
+ @Test
+ public void testStorageInvalid4() {
+ try {
+ String strStorage = "[";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+ CGSchema cgs1 = cgschemas[0];
+ System.out.println(cgs1);
+ } catch (Exception e) {
+ String errMsg = e.getMessage();
+ String str = "Encountered \" \"[\" \"[ \"\" at line 1, column 1.\nWas expecting one of:\n \"compress by\" ...\n \"serialize by\" ...\n ";
+ System.out.println(errMsg);
+ System.out.println(str);
+ Assert.assertEquals(errMsg, str);
+ }
+ }
+
+ @Test
+ public void testStorageInvalid5() {
+ try {
+ String strStorage = "[f1, f2]; [f1, f4]";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+ CGSchema cgs1 = cgschemas[0];
+ System.out.println(cgs1);
+ } catch (Exception e) {
+ String errMsg = e.getMessage();
+ String str = "Column f1 specified more than once!";
+ System.out.println(errMsg);
+ System.out.println(str);
+ Assert.assertEquals(errMsg, str);
+ }
+ }
+
+ @Test
+ public void testStorageInvalid6() {
+ try {
+ String strStorage = ":";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+ CGSchema cgs1 = cgschemas[0];
+ System.out.println(cgs1);
+ } catch (Exception e) {
+ String errMsg = e.getMessage();
+ String str = "Lexical error at line 1, column 2. Encountered: <EOF> after : \"\"";
+ System.out.println(errMsg);
+ System.out.println(str);
+ Assert.assertEquals(errMsg, str);
+ }
+ }
+
+ @Test
+ public void testStorageInvalid7() {
+ try {
+ String strStorage = "[f1, f2] serialize by xyz compress by gzip; [f3, f4] SERIALIZE BY avro COMPRESS BY lzo2";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+ CGSchema cgs1 = cgschemas[0];
+ System.out.println(cgs1);
+ } catch (Exception e) {
+ String errMsg = e.getMessage();
+ String str = "Encountered \" \"serialize by\" \"serialize by \"\" at line 1, column 10.";
+ System.out.println(errMsg);
+ System.out.println(str);
+ Assert.assertEquals(errMsg.startsWith(str), true);
+ }
+ }
+
+ @Test
+ public void testStorageInvalid8() {
+ try {
+ String strStorage = "[f1, f2] serialize by avro compress by xyz; [f3, f4] SERIALIZE BY avro COMPRESS BY lzo2";
+ Partition p = new Partition(schema.toString(), strStorage);
+ CGSchema[] cgschemas = p.getCGSchemas();
+ CGSchema cgs1 = cgschemas[0];
+ System.out.println(cgs1);
+ } catch (Exception e) {
+ String errMsg = e.getMessage();
+ String str = "Encountered \" \"compress by\" \"compress by \"\" at line 1, column 28.";
+ System.out.println(errMsg);
+ System.out.println(str);
+ Assert.assertEquals(errMsg.startsWith(str), true);
+ }
+ }
+}
\ No newline at end of file