You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by hu...@apache.org on 2019/01/16 17:32:03 UTC
incubator-gobblin git commit: [GOBBLIN-660] Fix OracleExtractor
datatype mapping
Repository: incubator-gobblin
Updated Branches:
refs/heads/master 0b1c52cd1 -> dc2d9e891
[GOBBLIN-660] Fix OracleExtractor datatype mapping
Closes #2531 from FrancisLfg/master
Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/dc2d9e89
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/dc2d9e89
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/dc2d9e89
Branch: refs/heads/master
Commit: dc2d9e891b095e3b18ebf7efc1cd57d65d06013e
Parents: 0b1c52c
Author: Francis Laforge <13...@users.noreply.github.com>
Authored: Wed Jan 16 09:31:56 2019 -0800
Committer: Hung Tran <hu...@linkedin.com>
Committed: Wed Jan 16 09:31:56 2019 -0800
----------------------------------------------------------------------
.../parquet/JsonElementConversionFactory.java | 4 ++++
.../gobblin/converter/parquet/JsonSchema.java | 2 +-
.../JsonIntermediateToParquetConverter.json | 20 +++++++++++++++++---
.../gobblin/source/jdbc/OracleExtractor.java | 2 +-
.../source/jdbc/OracleExtractorTest.java | 9 ++++-----
5 files changed, 27 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/dc2d9e89/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonElementConversionFactory.java
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonElementConversionFactory.java b/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonElementConversionFactory.java
index 1d3636a..46b264a 100644
--- a/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonElementConversionFactory.java
+++ b/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonElementConversionFactory.java
@@ -105,6 +105,10 @@ public class JsonElementConversionFactory {
case MAP:
return new MapConverter(schema);
+ case DATE:
+ case TIMESTAMP:
+ return new StringConverter(schema, repeated);
+
default:
throw new UnsupportedOperationException(fieldType + " is unsupported");
}
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/dc2d9e89/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonSchema.java
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonSchema.java b/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonSchema.java
index b7e001b..dc650a7 100644
--- a/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonSchema.java
+++ b/gobblin-modules/gobblin-parquet/src/main/java/org/apache/gobblin/converter/parquet/JsonSchema.java
@@ -55,7 +55,7 @@ public class JsonSchema extends Schema {
private final InputType type;
public enum InputType {
- STRING, INT, LONG, FLOAT, DOUBLE, BOOLEAN, ARRAY, ENUM, RECORD, MAP
+ STRING, INT, LONG, FLOAT, DOUBLE, BOOLEAN, ARRAY, ENUM, RECORD, MAP, DATE, TIMESTAMP
}
public JsonSchema(JsonArray jsonArray) {
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/dc2d9e89/gobblin-modules/gobblin-parquet/src/test/resources/converter/JsonIntermediateToParquetConverter.json
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-parquet/src/test/resources/converter/JsonIntermediateToParquetConverter.json b/gobblin-modules/gobblin-parquet/src/test/resources/converter/JsonIntermediateToParquetConverter.json
index e12325d..a588ea9 100644
--- a/gobblin-modules/gobblin-parquet/src/test/resources/converter/JsonIntermediateToParquetConverter.json
+++ b/gobblin-modules/gobblin-parquet/src/test/resources/converter/JsonIntermediateToParquetConverter.json
@@ -5,7 +5,9 @@
"b": 5.0,
"c": 8.0,
"d": true,
- "e": "somestring"
+ "e": "somestring",
+ "f": "2018-01-01",
+ "g": 1545083047
},
"schema": [
{
@@ -37,10 +39,22 @@
"dataType": {
"type": "string"
}
+ },
+ {
+ "columnName": "f",
+ "dataType": {
+ "type": "date"
+ }
+ },
+ {
+ "columnName": "g",
+ "dataType": {
+ "type": "timestamp"
+ }
}
],
- "expectedRecord": "a: 5 ; b: 5.0 ; c: 8.0 ; d: true ; e: somestring ; ",
- "expectedSchema": "message test_table{ ; required int32 a ; ; required float b ; ; required double c ; ; required boolean d ; ; required binary e (UTF8) ; ; } ; "
+ "expectedRecord": "a: 5 ; b: 5.0 ; c: 8.0 ; d: true ; e: somestring ; f: 2018-01-01 ; g: 1545083047 ;",
+ "expectedSchema": "message test_table{ ; required int32 a ; ; required float b ; ; required double c ; ; required boolean d ; ; required binary e (UTF8) ; ; required binary f (UTF8) ; ; required binary g (UTF8) ; ; } ; "
},
"array": {
"record": {
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/dc2d9e89/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/OracleExtractor.java
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/OracleExtractor.java b/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/OracleExtractor.java
index e893ef4..01001eb 100644
--- a/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/OracleExtractor.java
+++ b/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/OracleExtractor.java
@@ -67,7 +67,7 @@ public class OracleExtractor extends JdbcExtractor {
private static final String METADATA_SCHEMA_PSTMT_FORMAT =
"SELECT " +
"column_name, " +
- "UPPER(data_type), " +
+ "LOWER(data_type), " +
"NVL(data_length, 0) as length, " +
"NVL(data_precision, 0) as precesion, " +
"NVL(data_scale, 0) as scale, " +
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/dc2d9e89/gobblin-modules/gobblin-sql/src/test/java/org/apache/gobblin/source/jdbc/OracleExtractorTest.java
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-sql/src/test/java/org/apache/gobblin/source/jdbc/OracleExtractorTest.java b/gobblin-modules/gobblin-sql/src/test/java/org/apache/gobblin/source/jdbc/OracleExtractorTest.java
index ab82a07..6aa914a 100644
--- a/gobblin-modules/gobblin-sql/src/test/java/org/apache/gobblin/source/jdbc/OracleExtractorTest.java
+++ b/gobblin-modules/gobblin-sql/src/test/java/org/apache/gobblin/source/jdbc/OracleExtractorTest.java
@@ -69,18 +69,17 @@ public class OracleExtractorTest {
}
@Test
- public void testConstructSampleClause() throws Exception {
+ public void testConstructSampleClause() {
String sClause = oracleExtractor.constructSampleClause();
assertEquals(sClause.trim(), (" rownum <= " + oracleExtractor.getSampleRecordCount()).trim());
}
@Test
- public void testRemoveSampleClauseFromQuery() throws Exception {
+ public void testRemoveSampleClauseFromQuery() {
String q1Expected = "SELECT * FROM x WHERE 1=1";
String q2Expected = "SELECT * FROM x WHERE 1=1 AND x.a < 10";
String q3Expected = "SELECT * FROM x WHERE x.a < 10 AND 1=1";
String q4Expected = "SELECT * FROM x WHERE x.a < 10 AND 1=1 AND x.b = 20";
- String qEmptyClean = "";
String q1Parsed = oracleExtractor.removeSampleClauseFromQuery(QUERY_1);
String q2Parsed = oracleExtractor.removeSampleClauseFromQuery(QUERY_2);
@@ -94,7 +93,7 @@ public class OracleExtractorTest {
}
@Test
- public void testExractSampleRecordCountFromQuery() throws Exception {
+ public void testExractSampleRecordCountFromQuery() {
long res1 = oracleExtractor.exractSampleRecordCountFromQuery(QUERY_1);
long res2 = oracleExtractor.exractSampleRecordCountFromQuery(QUERY_2);
long res3 = oracleExtractor.exractSampleRecordCountFromQuery(QUERY_3);
@@ -113,7 +112,7 @@ public class OracleExtractorTest {
/**
* Build a mock implementation of Result using Mockito
*/
- private ResultSet buildMockResultSet() throws Exception {
+ private ResultSet buildMockResultSet() {
MockResultSet mrs = new MockResultSet(StringUtils.EMPTY);
for (MockJdbcColumn column : COLUMNS) {