You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by da...@apache.org on 2016/07/20 22:08:58 UTC
hive git commit: HIVE-14282: HCatLoader ToDate() exception with hive
partition table , partitioned by column of DATE datatype (Daniel Dai,
reviewed by Thejas Nair)
Repository: hive
Updated Branches:
refs/heads/master 7e4cb161a -> d868b6094
HIVE-14282: HCatLoader ToDate() exception with hive partition table ,partitioned by column of DATE datatype (Daniel Dai, reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d868b609
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d868b609
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d868b609
Branch: refs/heads/master
Commit: d868b60944b59a03dfb3f7c5a00cf5cf93a7bd95
Parents: 7e4cb16
Author: Daniel Dai <da...@hortonworks.com>
Authored: Wed Jul 20 15:08:14 2016 -0700
Committer: Daniel Dai <da...@hortonworks.com>
Committed: Wed Jul 20 15:08:14 2016 -0700
----------------------------------------------------------------------
.../apache/hive/hcatalog/pig/HCatLoader.java | 34 ++++++++++++++++++
.../hive/hcatalog/pig/TestHCatLoader.java | 36 ++++++++++++++++++++
pom.xml | 2 +-
3 files changed, 71 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d868b609/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java
----------------------------------------------------------------------
diff --git a/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java b/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java
index bb5dd4f..d98f40c 100644
--- a/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java
+++ b/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java
@@ -38,18 +38,23 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.security.Credentials;
import org.apache.hive.hcatalog.common.HCatConstants;
import org.apache.hive.hcatalog.common.HCatContext;
+import org.apache.hive.hcatalog.common.HCatException;
import org.apache.hive.hcatalog.common.HCatUtil;
import org.apache.hive.hcatalog.data.Pair;
+import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hive.hcatalog.data.schema.HCatSchema;
import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
import org.apache.hive.hcatalog.mapreduce.InputJobInfo;
import org.apache.hive.hcatalog.mapreduce.SpecialCases;
import org.apache.pig.Expression;
import org.apache.pig.Expression.BinaryExpression;
+import org.apache.pig.Expression.Const;
import org.apache.pig.PigException;
import org.apache.pig.ResourceSchema;
import org.apache.pig.ResourceStatistics;
import org.apache.pig.impl.util.UDFContext;
+import org.joda.time.DateTime;
+import org.joda.time.format.DateTimeFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -273,6 +278,16 @@ public class HCatLoader extends HCatBaseLoader {
return partitionFilterString;
}
+ private String getHCatConstString(Const con, HCatFieldSchema.Type type) {
+ Object value = con.getValue();
+ switch (type) {
+ case DATE:
+ return ((DateTime)value).toString(DateTimeFormat.forPattern("YYYY-MM-dd"));
+ default:
+ return con.toString();
+ }
+ }
+
private String getHCatComparisonString(Expression expr) {
if (expr instanceof BinaryExpression) {
// call getHCatComparisonString on lhs and rhs, and and join the
@@ -290,6 +305,25 @@ public class HCatLoader extends HCatBaseLoader {
opStr = expr.getOpType().toString();
}
BinaryExpression be = (BinaryExpression) expr;
+ if (be.getRhs() instanceof Const) {
+ // If the expr is column op const, will try to cast the const to string
+ // according to the data type of the column
+ UDFContext udfContext = UDFContext.getUDFContext();
+ Properties udfProps = udfContext.getUDFProperties(this.getClass(),
+ new String[]{signature});
+ HCatSchema hcatTableSchema = (HCatSchema) udfProps.get(HCatConstants.HCAT_TABLE_SCHEMA);
+ HCatFieldSchema fs = null;
+ try {
+ fs = hcatTableSchema.get(be.getLhs().toString());
+ } catch (HCatException e) {
+ // Shall never happen
+ }
+ if (fs != null) {
+ return "(" + getHCatComparisonString(be.getLhs()) +
+ opStr +
+ getHCatConstString((Const)be.getRhs(), fs.getType()) + ")";
+ }
+ }
return "(" + getHCatComparisonString(be.getLhs()) +
opStr +
getHCatComparisonString(be.getRhs()) + ")";
http://git-wip-us.apache.org/repos/asf/hive/blob/d868b609/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
----------------------------------------------------------------------
diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
index 678339c..71b09b0 100644
--- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
+++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
@@ -87,11 +87,13 @@ public class TestHCatLoader {
private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data";
private static final String COMPLEX_FILE_NAME = TEST_DATA_DIR + "/complex.input.data";
+ private static final String DATE_FILE_NAME = TEST_DATA_DIR + "/datetimestamp.input.data";
private static final String BASIC_TABLE = "junit_unparted_basic";
private static final String COMPLEX_TABLE = "junit_unparted_complex";
private static final String PARTITIONED_TABLE = "junit_parted_basic";
private static final String SPECIFIC_SIZE_TABLE = "junit_specific_size";
+ private static final String PARTITIONED_DATE_TABLE = "junit_parted_date";
private Driver driver;
private Map<Integer, Pair<Integer, String>> basicInputData;
@@ -104,6 +106,8 @@ public class TestHCatLoader {
add("testProjectionsBasic");
add("testColumnarStorePushdown2");
add("testReadMissingPartitionBasicNeg");
+ add("testDatePartitionPushUp");
+ add("testTimestampPartitionPushUp");
}});
}};
@@ -200,6 +204,7 @@ public class TestHCatLoader {
createTable(PARTITIONED_TABLE, "a int, b string", "bkt string");
createTable(SPECIFIC_SIZE_TABLE, "a int, b string");
+ createTable(PARTITIONED_DATE_TABLE, "b string", "dt date");
AllTypesTable.setupAllTypesTable(driver);
int LOOP_SIZE = 3;
@@ -222,6 +227,12 @@ public class TestHCatLoader {
"Edward Hyde\t1337\t(415-253-6367,anonymous@b44chan.org)\t{(CREATIVE_WRITING),(COPYRIGHT_LAW)}\t[CREATIVE_WRITING#A+,COPYRIGHT_LAW#D]\t{(415-253-6367,cell),(408-253-6367,landline)}",
}
);
+ HcatTestUtils.createTestDataFile(DATE_FILE_NAME,
+ new String[]{
+ "2016-07-14 08:10:15\tHenry Jekyll",
+ "2016-07-15 11:54:55\tEdward Hyde",
+ }
+ );
PigServer server = new PigServer(ExecType.LOCAL);
server.setBatchOn();
int i = 0;
@@ -239,6 +250,11 @@ public class TestHCatLoader {
server.registerQuery("D = load '" + COMPLEX_FILE_NAME + "' as (name:chararray, studentid:int, contact:tuple(phno:chararray,email:chararray), currently_registered_courses:bag{innertup:tuple(course:chararray)}, current_grades:map[ ] , phnos :bag{innertup:tuple(phno:chararray,type:chararray)});", ++i);
server.registerQuery("store D into '" + COMPLEX_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();", ++i);
+
+ server.registerQuery("E = load '" + DATE_FILE_NAME + "' as (dt:chararray, b:chararray);", ++i);
+ server.registerQuery("F = foreach E generate ToDate(dt, 'yyyy-MM-dd HH:mm:ss') as dt, b;", ++i);
+ server.registerQuery("store F into '" + PARTITIONED_DATE_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();", ++i);
+
server.executeBatch();
}
@@ -250,6 +266,7 @@ public class TestHCatLoader {
dropTable(COMPLEX_TABLE);
dropTable(PARTITIONED_TABLE);
dropTable(SPECIFIC_SIZE_TABLE);
+ dropTable(PARTITIONED_DATE_TABLE);
dropTable(AllTypesTable.ALL_PRIMITIVE_TYPES_TABLE);
}
} finally {
@@ -661,6 +678,25 @@ public class TestHCatLoader {
}
/**
+ * Test if we can read a date partitioned table
+ */
+ @Test
+ public void testDatePartitionPushUp() throws Exception {
+ assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS));
+ PigServer server = new PigServer(ExecType.LOCAL);
+ server.registerQuery("X = load '" + PARTITIONED_DATE_TABLE + "' using " + HCatLoader.class.getName() + "();");
+ server.registerQuery("Y = filter X by dt == ToDate('2016-07-14','yyyy-MM-dd');");
+ Iterator<Tuple> YIter = server.openIterator("Y");
+ int numTuplesRead = 0;
+ while (YIter.hasNext()) {
+ Tuple t = YIter.next();
+ assertEquals(t.size(), 2);
+ numTuplesRead++;
+ }
+ assertTrue("Expected " + 1 + "; found " + numTuplesRead, numTuplesRead == 1);
+ }
+
+ /**
* basic tests that cover each scalar type
* https://issues.apache.org/jira/browse/HIVE-5814
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/d868b609/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7d22b4a..d185e34 100644
--- a/pom.xml
+++ b/pom.xml
@@ -168,7 +168,7 @@
<mina.version>2.0.0-M5</mina.version>
<netty.version>4.0.23.Final</netty.version>
<parquet.version>1.8.1</parquet.version>
- <pig.version>0.12.0</pig.version>
+ <pig.version>0.16.0</pig.version>
<protobuf.version>2.5.0</protobuf.version>
<stax.version>1.0.1</stax.version>
<slf4j.version>1.7.10</slf4j.version>