You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2014/09/19 00:22:55 UTC
svn commit: r1626093 -
/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/
Author: jdere
Date: Thu Sep 18 22:22:54 2014
New Revision: 1626093
URL: http://svn.apache.org/r1626093
Log:
HIVE-7457: Minor HCatalog Pig Adapter test clean up (David Chen via Jason Dere)
Added:
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorerMulti.java
Removed:
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatPigStorer.java
Modified:
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/MockLoader.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorer.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestPigHCatUtil.java
Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/MockLoader.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/MockLoader.java?rev=1626093&r1=1626092&r2=1626093&view=diff
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/MockLoader.java (original)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/MockLoader.java Thu Sep 18 22:22:54 2014
@@ -36,6 +36,7 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
import org.apache.pig.LoadFunc;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
import org.apache.pig.data.Tuple;
Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java?rev=1626093&r1=1626092&r2=1626093&view=diff
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java (original)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java Thu Sep 18 22:22:54 2014
@@ -23,9 +23,8 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
-import junit.framework.TestCase;
-
import org.apache.commons.io.FileUtils;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hive.cli.CliSessionState;
@@ -42,6 +41,7 @@ import org.apache.hadoop.mapreduce.Recor
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskAttemptID;
+
import org.apache.hive.hcatalog.HcatTestUtils;
import org.apache.hive.hcatalog.common.HCatConstants;
import org.apache.hive.hcatalog.common.HCatContext;
@@ -51,12 +51,16 @@ import org.apache.hive.hcatalog.mapreduc
import org.apache.hive.hcatalog.mapreduce.HCatOutputFormat;
import org.apache.hive.hcatalog.mapreduce.OutputJobInfo;
import org.apache.hive.hcatalog.mapreduce.HCatMapRedUtil;
+
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.data.Tuple;
-public class TestE2EScenarios extends TestCase {
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+public class TestE2EScenarios {
private static final String TEST_DATA_DIR = System.getProperty("java.io.tmpdir") + File.separator
+ TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis();
private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
@@ -69,9 +73,8 @@ public class TestE2EScenarios extends Te
return "orc";
}
- @Override
- protected void setUp() throws Exception {
-
+ @Before
+ public void setUp() throws Exception {
File f = new File(TEST_WAREHOUSE_DIR);
if (f.exists()) {
FileUtil.fullyDelete(f);
@@ -90,8 +93,8 @@ public class TestE2EScenarios extends Te
}
- @Override
- protected void tearDown() throws Exception {
+ @After
+ public void tearDown() throws Exception {
try {
dropTable("inpy");
dropTable("rc5318");
@@ -146,16 +149,13 @@ public class TestE2EScenarios extends Te
System.err.println("===");
}
-
private void copyTable(String in, String out) throws IOException, InterruptedException {
Job ijob = new Job();
Job ojob = new Job();
HCatInputFormat inpy = new HCatInputFormat();
inpy.setInput(ijob , null, in);
HCatOutputFormat oupy = new HCatOutputFormat();
- oupy.setOutput(ojob,
- OutputJobInfo.create(null, out, new HashMap<String,String>()
- ));
+ oupy.setOutput(ojob, OutputJobInfo.create(null, out, new HashMap<String,String>()));
// Test HCatContext
@@ -207,6 +207,7 @@ public class TestE2EScenarios extends Te
}
+ @Test
public void testReadOrcAndRCFromPig() throws Exception {
String tableSchema = "ti tinyint, si smallint,i int, bi bigint, f float, d double, b boolean";
@@ -224,15 +225,14 @@ public class TestE2EScenarios extends Te
driverRun("LOAD DATA LOCAL INPATH '"+TEXTFILE_LOCN+"' OVERWRITE INTO TABLE inpy");
// write it out from hive to an rcfile table, and to an orc table
-// driverRun("insert overwrite table rc5318 select * from inpy");
+ //driverRun("insert overwrite table rc5318 select * from inpy");
copyTable("inpy","rc5318");
-// driverRun("insert overwrite table orc5318 select * from inpy");
+ //driverRun("insert overwrite table orc5318 select * from inpy");
copyTable("inpy","orc5318");
pigDump("inpy");
pigDump("rc5318");
pigDump("orc5318");
-
}
}
Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java?rev=1626093&r1=1626092&r2=1626093&view=diff
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java (original)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java Thu Sep 18 22:22:54 2014
@@ -34,6 +34,7 @@ import java.util.Map;
import java.util.Properties;
import org.apache.commons.io.FileUtils;
+
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
@@ -45,11 +46,13 @@ import org.apache.hadoop.hive.ql.process
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.mapreduce.Job;
+
import org.apache.hive.hcatalog.HcatTestUtils;
import org.apache.hive.hcatalog.common.HCatUtil;
import org.apache.hive.hcatalog.common.HCatConstants;
import org.apache.hive.hcatalog.data.Pair;
import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
+
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.ResourceStatistics;
@@ -60,10 +63,13 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.PigRunner;
import org.apache.pig.tools.pigstats.OutputStats;
import org.apache.pig.tools.pigstats.PigStats;
+
import org.joda.time.DateTime;
+
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -72,7 +78,7 @@ import static org.junit.Assert.*;
public class TestHCatLoader {
private static final Logger LOG = LoggerFactory.getLogger(TestHCatLoader.class);
private static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty("java.io.tmpdir") +
- File.separator + TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis());
+ File.separator + TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis());
private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data";
private static final String COMPLEX_FILE_NAME = TEST_DATA_DIR + "/complex.input.data";
@@ -93,6 +99,7 @@ public class TestHCatLoader {
private void dropTable(String tablename) throws IOException, CommandNeedRetryException {
dropTable(tablename, driver);
}
+
static void dropTable(String tablename, Driver driver) throws IOException, CommandNeedRetryException {
driver.run("drop table if exists " + tablename);
}
@@ -100,7 +107,8 @@ public class TestHCatLoader {
private void createTable(String tablename, String schema, String partitionedBy) throws IOException, CommandNeedRetryException {
createTable(tablename, schema, partitionedBy, driver, storageFormat());
}
- static void createTable(String tablename, String schema, String partitionedBy, Driver driver, String storageFormat)
+
+ static void createTable(String tablename, String schema, String partitionedBy, Driver driver, String storageFormat)
throws IOException, CommandNeedRetryException {
String createTable;
createTable = "create table " + tablename + "(" + schema + ") ";
@@ -114,6 +122,7 @@ public class TestHCatLoader {
private void createTable(String tablename, String schema) throws IOException, CommandNeedRetryException {
createTable(tablename, schema, null);
}
+
/**
* Execute Hive CLI statement
* @param cmd arbitrary statement to execute
@@ -125,20 +134,20 @@ public class TestHCatLoader {
throw new IOException("Failed to execute \"" + cmd + "\". Driver returned " + cpr.getResponseCode() + " Error: " + cpr.getErrorMessage());
}
}
+
private static void checkProjection(FieldSchema fs, String expectedName, byte expectedPigType) {
assertEquals(fs.alias, expectedName);
assertEquals("Expected " + DataType.findTypeName(expectedPigType) + "; got " +
DataType.findTypeName(fs.type), expectedPigType, fs.type);
}
-
+
@Before
public void setup() throws Exception {
-
File f = new File(TEST_WAREHOUSE_DIR);
if (f.exists()) {
FileUtil.fullyDelete(f);
}
- if(!(new File(TEST_WAREHOUSE_DIR).mkdirs())) {
+ if (!(new File(TEST_WAREHOUSE_DIR).mkdirs())) {
throw new RuntimeException("Could not create " + TEST_WAREHOUSE_DIR);
}
@@ -192,7 +201,7 @@ public class TestHCatLoader {
server.registerQuery("B = foreach A generate a,b;", ++i);
server.registerQuery("B2 = filter B by a < 2;", ++i);
server.registerQuery("store B2 into '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer('bkt=0');", ++i);
-
+
server.registerQuery("C = foreach A generate a,b;", ++i);
server.registerQuery("C2 = filter C by a >= 2;", ++i);
server.registerQuery("store C2 into '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer('bkt=1');", ++i);
@@ -470,7 +479,7 @@ public class TestHCatLoader {
{
fs.delete(new Path(PIGOUTPUT_DIR), true);
}
- }finally {
+ } finally {
new File(PIG_FILE).delete();
}
}
@@ -534,7 +543,7 @@ public class TestHCatLoader {
}
/**
- * basic tests that cover each scalar type
+ * basic tests that cover each scalar type
* https://issues.apache.org/jira/browse/HIVE-5814
*/
private static final class AllTypesTable {
Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java?rev=1626093&r1=1626092&r2=1626093&view=diff
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java (original)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java Thu Sep 18 22:22:54 2014
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.ql.Command
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
+
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
@@ -44,8 +45,10 @@ import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
+
import org.junit.BeforeClass;
import org.junit.Test;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -225,7 +228,7 @@ public class TestHCatLoaderComplexSchema
dropTable(tablename);
}
}
-
+
private void compareTuples(Tuple t1, Tuple t2) throws ExecException {
Assert.assertEquals("Tuple Sizes don't match", t1.size(), t2.size());
for (int i = 0; i < t1.size(); i++) {
@@ -237,7 +240,7 @@ public class TestHCatLoaderComplexSchema
Assert.assertEquals(msg, noOrder(f1.toString()), noOrder(f2.toString()));
}
}
-
+
private String noOrder(String s) {
char[] chars = s.toCharArray();
Arrays.sort(chars);
Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java?rev=1626093&r1=1626092&r2=1626093&view=diff
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java (original)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java Thu Sep 18 22:22:54 2014
@@ -31,8 +31,10 @@ import java.util.Properties;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+
import org.apache.hive.hcatalog.HcatTestUtils;
import org.apache.hive.hcatalog.mapreduce.HCatBaseTest;
+
import org.apache.pig.EvalFunc;
import org.apache.pig.ExecType;
import org.apache.pig.PigException;
@@ -41,10 +43,13 @@ import org.apache.pig.data.DataByteArray
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.util.LogUtils;
+
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
+
import org.junit.Assert;
import org.junit.Test;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,6 +68,7 @@ public class TestHCatStorer extends HCat
pigValueRangeTestOverflow("junitTypeTest3", "tinyint", "int", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
Integer.toString(300));
}
+
@Test
public void testWriteSmallint() throws Exception {
pigValueRangeTest("junitTypeTest1", "smallint", "int", null, Integer.toString(Short.MIN_VALUE),
@@ -72,6 +78,7 @@ public class TestHCatStorer extends HCat
pigValueRangeTestOverflow("junitTypeTest3", "smallint", "int", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
Integer.toString(Short.MAX_VALUE + 1));
}
+
@Test
public void testWriteChar() throws Exception {
pigValueRangeTest("junitTypeTest1", "char(5)", "chararray", null, "xxx", "xxx ");
@@ -81,6 +88,7 @@ public class TestHCatStorer extends HCat
pigValueRangeTestOverflow("junitTypeTest3", "char(5)", "chararray", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
"too_long2");
}
+
@Test
public void testWriteVarchar() throws Exception {
pigValueRangeTest("junitTypeTest1", "varchar(5)", "chararray", null, "xxx", "xxx");
@@ -90,6 +98,7 @@ public class TestHCatStorer extends HCat
pigValueRangeTestOverflow("junitTypeTest3", "varchar(5)", "chararray", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
"too_long2");
}
+
@Test
public void testWriteDecimalXY() throws Exception {
pigValueRangeTest("junitTypeTest1", "decimal(5,2)", "bigdecimal", null, BigDecimal.valueOf(1.2).toString(),
@@ -100,6 +109,7 @@ public class TestHCatStorer extends HCat
pigValueRangeTestOverflow("junitTypeTest3", "decimal(5,2)", "bigdecimal", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
BigDecimal.valueOf(500.123).toString());
}
+
@Test
public void testWriteDecimalX() throws Exception {
//interestingly decimal(2) means decimal(2,0)
@@ -110,6 +120,7 @@ public class TestHCatStorer extends HCat
pigValueRangeTestOverflow("junitTypeTest3", "decimal(2)", "bigdecimal", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
BigDecimal.valueOf(50.123).toString());
}
+
@Test
public void testWriteDecimal() throws Exception {
//decimal means decimal(10,0)
@@ -120,9 +131,10 @@ public class TestHCatStorer extends HCat
pigValueRangeTestOverflow("junitTypeTest3", "decimal", "bigdecimal", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
BigDecimal.valueOf(12345678900L).toString());
}
+
/**
* because we want to ignore TZ which is included in toString()
- * include time to make sure it's 0
+ * include time to make sure it's 0
*/
private static final String FORMAT_4_DATE = "yyyy-MM-dd HH:mm:ss";
@Test
@@ -142,6 +154,7 @@ public class TestHCatStorer extends HCat
pigValueRangeTestOverflow("junitTypeTest6", "date", "datetime", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
d.plusMinutes(1).toString(), FORMAT_4_DATE);//date out of range due to time!=0
}
+
@Test
public void testWriteDate3() throws Exception {
DateTime d = new DateTime(1991,10,11,23,10,DateTimeZone.forOffsetHours(-11));
@@ -154,6 +167,7 @@ public class TestHCatStorer extends HCat
pigValueRangeTestOverflow("junitTypeTest6", "date", "datetime", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
d.plusMinutes(1).toString(), FORMAT_4_DATE);
}
+
@Test
public void testWriteDate2() throws Exception {
DateTime d = new DateTime(1991,11,12,0,0, DateTimeZone.forID("US/Eastern"));
@@ -168,46 +182,48 @@ public class TestHCatStorer extends HCat
pigValueRangeTestOverflow("junitTypeTest3", "date", "datetime", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
d.plusMinutes(1).toString(), FORMAT_4_DATE);
}
+
/**
- * Note that the value that comes back from Hive will have local TZ on it. Using local is
+ * Note that the value that comes back from Hive will have local TZ on it. Using local is
* arbitrary but DateTime needs TZ (or will assume default) and Hive does not have TZ.
* So if you start with Pig value in TZ=x and write to Hive, when you read it back the TZ may
* be different. The millis value should match, of course.
- *
+ *
* @throws Exception
*/
@Test
public void testWriteTimestamp() throws Exception {
DateTime d = new DateTime(1991,10,11,14,23,30, 10);//uses default TZ
- pigValueRangeTest("junitTypeTest1", "timestamp", "datetime", null, d.toString(),
+ pigValueRangeTest("junitTypeTest1", "timestamp", "datetime", null, d.toString(),
d.toDateTime(DateTimeZone.getDefault()).toString());
d = d.plusHours(2);
pigValueRangeTest("junitTypeTest2", "timestamp", "datetime", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Null,
d.toString(), d.toDateTime(DateTimeZone.getDefault()).toString());
d = d.toDateTime(DateTimeZone.UTC);
- pigValueRangeTest("junitTypeTest3", "timestamp", "datetime", null, d.toString(),
+ pigValueRangeTest("junitTypeTest3", "timestamp", "datetime", null, d.toString(),
d.toDateTime(DateTimeZone.getDefault()).toString());
d = new DateTime(1991,10,11,23,24,25, 26);
- pigValueRangeTest("junitTypeTest1", "timestamp", "datetime", null, d.toString(),
+ pigValueRangeTest("junitTypeTest1", "timestamp", "datetime", null, d.toString(),
d.toDateTime(DateTimeZone.getDefault()).toString());
d = d.toDateTime(DateTimeZone.UTC);
- pigValueRangeTest("junitTypeTest3", "timestamp", "datetime", null, d.toString(),
+ pigValueRangeTest("junitTypeTest3", "timestamp", "datetime", null, d.toString(),
d.toDateTime(DateTimeZone.getDefault()).toString());
}
//End: tests that check values from Pig that are out of range for target column
-
private void pigValueRangeTestOverflow(String tblName, String hiveType, String pigType,
HCatBaseStorer.OOR_VALUE_OPT_VALUES goal, String inputValue, String format) throws Exception {
pigValueRangeTest(tblName, hiveType, pigType, goal, inputValue, null, format);
}
+
private void pigValueRangeTestOverflow(String tblName, String hiveType, String pigType,
HCatBaseStorer.OOR_VALUE_OPT_VALUES goal, String inputValue) throws Exception {
pigValueRangeTest(tblName, hiveType, pigType, goal, inputValue, null, null);
}
+
private void pigValueRangeTest(String tblName, String hiveType, String pigType,
- HCatBaseStorer.OOR_VALUE_OPT_VALUES goal, String inputValue,
+ HCatBaseStorer.OOR_VALUE_OPT_VALUES goal, String inputValue,
String expectedValue) throws Exception {
pigValueRangeTest(tblName, hiveType, pigType, goal, inputValue, expectedValue, null);
}
@@ -218,6 +234,7 @@ public class TestHCatStorer extends HCat
String getStorageFormat() {
return "RCFILE";
}
+
/**
* This is used to test how Pig values of various data types which are out of range for Hive target
* column are handled. Currently the options are to raise an error or write NULL.
@@ -236,7 +253,7 @@ public class TestHCatStorer extends HCat
* @param format date format to use for comparison of values since default DateTime.toString()
* includes TZ which is meaningless for Hive DATE type
*/
- private void pigValueRangeTest(String tblName, String hiveType, String pigType,
+ private void pigValueRangeTest(String tblName, String hiveType, String pigType,
HCatBaseStorer.OOR_VALUE_OPT_VALUES goal, String inputValue, String expectedValue, String format)
throws Exception {
TestHCatLoader.dropTable(tblName, driver);
@@ -309,6 +326,7 @@ public class TestHCatStorer extends HCat
Unfortunately Timestamp.toString() adjusts the value for local TZ and 't' is a String
thus the timestamp in 't' doesn't match rawData*/
}
+
/**
* Create a data file with datatypes added in 0.13. Read it with Pig and use
* Pig + HCatStorer to write to a Hive table. Then read it using Pig and Hive
@@ -365,6 +383,7 @@ public class TestHCatStorer extends HCat
}
Assert.assertEquals("Expected " + NUM_ROWS + " rows; got " + numRowsRead + " file=" + INPUT_FILE_NAME, NUM_ROWS, numRowsRead);
}
+
static void dumpFile(String fileName) throws Exception {
File file = new File(fileName);
BufferedReader reader = new BufferedReader(new FileReader(file));
@@ -375,6 +394,7 @@ public class TestHCatStorer extends HCat
}
reader.close();
}
+
@Test
public void testPartColsInData() throws IOException, CommandNeedRetryException {
Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java?rev=1626093&r1=1626092&r2=1626093&view=diff
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java (original)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java Thu Sep 18 22:22:54 2014
@@ -25,20 +25,26 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
-import junit.framework.TestCase;
-
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.session.SessionState;
+
import org.apache.hive.hcatalog.common.HCatUtil;
import org.apache.hive.hcatalog.data.Pair;
+
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
-public class TestHCatStorerMulti extends TestCase {
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestHCatStorerMulti {
public static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(
System.getProperty("user.dir") + "/build/test/data/" +
TestHCatStorerMulti.class.getCanonicalName() + "-" + System.currentTimeMillis());
@@ -77,8 +83,8 @@ public class TestHCatStorerMulti extends
createTable(tablename, schema, null);
}
- @Override
- protected void setUp() throws Exception {
+ @Before
+ public void setUp() throws Exception {
if (driver == null) {
HiveConf hiveConf = new HiveConf(this.getClass());
hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
@@ -92,14 +98,13 @@ public class TestHCatStorerMulti extends
cleanup();
}
- @Override
- protected void tearDown() throws Exception {
+ @After
+ public void tearDown() throws Exception {
cleanup();
}
+ @Test
public void testStoreBasicTable() throws Exception {
-
-
createTable(BASIC_TABLE, "a int, b string");
populateBasicFile();
@@ -117,6 +122,7 @@ public class TestHCatStorerMulti extends
assertEquals(basicInputData.size(), unpartitionedTableValuesReadFromHiveDriver.size());
}
+ @Test
public void testStorePartitionedTable() throws Exception {
createTable(PARTITIONED_TABLE, "a int, b string", "bkt string");
@@ -139,9 +145,8 @@ public class TestHCatStorerMulti extends
assertEquals(basicInputData.size(), partitionedTableValuesReadFromHiveDriver.size());
}
+ @Test
public void testStoreTableMulti() throws Exception {
-
-
createTable(BASIC_TABLE, "a int, b string");
createTable(PARTITIONED_TABLE, "a int, b string", "bkt string");
Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java?rev=1626093&r1=1626092&r2=1626093&view=diff
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java (original)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java Thu Sep 18 22:22:54 2014
@@ -25,10 +25,13 @@ import java.util.Iterator;
import java.util.UUID;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+
import org.apache.hive.hcatalog.HcatTestUtils;
import org.apache.hive.hcatalog.mapreduce.HCatBaseTest;
+
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
+
import org.junit.Assert;
import org.junit.Test;
Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorer.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorer.java?rev=1626093&r1=1626092&r2=1626093&view=diff
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorer.java (original)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorer.java Thu Sep 18 22:22:54 2014
@@ -18,11 +18,15 @@
*/
package org.apache.hive.hcatalog.pig;
-public class TestOrcHCatStorer extends TestHCatStorerMulti {
+import java.io.IOException;
- @Override
- protected String storageFormat() {
- return "orc";
+import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestOrcHCatStorer extends TestHCatStorer {
+ @Override String getStorageFormat() {
+ return "ORC";
}
}
-
Added: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorerMulti.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorerMulti.java?rev=1626093&view=auto
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorerMulti.java (added)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorerMulti.java Thu Sep 18 22:22:54 2014
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hive.hcatalog.pig;
+
+public class TestOrcHCatStorerMulti extends TestHCatStorerMulti {
+
+ @Override
+ protected String storageFormat() {
+ return "orc";
+ }
+}
+
Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestPigHCatUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestPigHCatUtil.java?rev=1626093&r1=1626092&r2=1626093&view=diff
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestPigHCatUtil.java (original)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestPigHCatUtil.java Thu Sep 18 22:22:54 2014
@@ -20,14 +20,18 @@
package org.apache.hive.hcatalog.pig;
import com.google.common.collect.Lists;
+
import junit.framework.Assert;
+
import org.apache.hive.hcatalog.common.HCatConstants;
import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hive.hcatalog.data.schema.HCatSchema;
+
import org.apache.pig.ResourceSchema;
import org.apache.pig.ResourceSchema.ResourceFieldSchema;
import org.apache.pig.data.DataType;
import org.apache.pig.impl.util.UDFContext;
+
import org.junit.Test;
public class TestPigHCatUtil {