You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by ab...@apache.org on 2018/03/21 08:30:41 UTC
[23/50] incubator-gobblin git commit: postgresql extractor fixes *
covering more data types * adding source.conn.database which allows you to
connect to a database that has a different name than the source schema (i.e.
this is 3-part naming convention).
postgresql extractor fixes
* covering more data types
* adding source.conn.database which allows you to connect to a database
that has a different name than the source schema (i.e. this is 3-part
naming convention). Without this you cannot connect to a database
"workgroup" to pull the table "public.http_requests"
Closes #2152 from DepthDeluxe/master
Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/34de6bfd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/34de6bfd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/34de6bfd
Branch: refs/heads/0.12.0
Commit: 34de6bfd3c06399a29937249390454fc5fa57f00
Parents: 3598d10
Author: Colin Heinzmann <ch...@linkedin.com>
Authored: Mon Feb 5 12:05:55 2018 -0800
Committer: Abhishek Tiwari <ab...@gmail.com>
Committed: Mon Feb 5 12:05:55 2018 -0800
----------------------------------------------------------------------
.../apache/gobblin/source/jdbc/PostgresqlExtractor.java | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/34de6bfd/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/PostgresqlExtractor.java
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/PostgresqlExtractor.java b/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/PostgresqlExtractor.java
index 33b4e00..6876739 100644
--- a/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/PostgresqlExtractor.java
+++ b/gobblin-modules/gobblin-sql/src/main/java/org/apache/gobblin/source/jdbc/PostgresqlExtractor.java
@@ -44,6 +44,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class PostgresqlExtractor extends JdbcExtractor {
+ private static final String CONNECTION_DATABASE = "source.conn.database";
private static final String POSTGRES_TIMESTAMP_FORMAT = "yyyy-MM-dd HH:mm:ss";
private static final String POSTGRES_DATE_FORMAT = "yyyy-MM-dd";
private static final String POSTGRES_HOUR_FORMAT = "HH";
@@ -168,7 +169,8 @@ public class PostgresqlExtractor extends JdbcExtractor {
public String getConnectionUrl() {
String host = this.workUnitState.getProp(ConfigurationKeys.SOURCE_CONN_HOST_NAME);
String port = this.workUnitState.getProp(ConfigurationKeys.SOURCE_CONN_PORT);
- String database = this.workUnitState.getProp(ConfigurationKeys.SOURCE_QUERYBASED_SCHEMA);
+ String database = this.workUnitState.getProp(CONNECTION_DATABASE);
+
return "jdbc:postgresql://" + host.trim() + ":" + port + "/" + database.trim();
}
@@ -182,13 +184,15 @@ public class PostgresqlExtractor extends JdbcExtractor {
public Map<String, String> getDataTypeMap() {
Map<String, String> dataTypeMap =
ImmutableMap.<String, String>builder().put("tinyint", "int").put("smallint", "int").put("mediumint", "int")
- .put("int", "int").put("bigint", "long").put("float", "float").put("double", "double")
- .put("decimal", "double").put("numeric", "double").put("date", "date").put("timestamp", "timestamp")
+ .put("integer", "int").put("int", "int").put("bigint", "long").put("float", "float").put("double", "double")
+ .put("double precision", "double").put("decimal", "double").put("numeric", "double").put("date", "date").put("timestamp", "timestamp")
+ .put("timestamp without time zone", "timestamp").put("timestamp with time zone", "timestamp")
.put("datetime", "timestamp").put("time", "time").put("char", "string").put("varchar", "string")
.put("varbinary", "string").put("text", "string").put("tinytext", "string").put("mediumtext", "string")
- .put("longtext", "string").put("blob", "string").put("tinyblob", "string").put("mediumblob", "string")
+ .put("character varying", "string").put("longtext", "string").put("blob", "string").put("tinyblob", "string").put("mediumblob", "string")
.put("longblob", "string").put("enum", "string").build();
return dataTypeMap;
+
}
@Override