You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by od...@apache.org on 2017/01/24 02:32:40 UTC
[2/4] incubator-hawq git commit: HAWQ-1228. Hardcoded comma as
delimiter.
HAWQ-1228. Hardcoded comma as delimiter.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/8fe27032
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/8fe27032
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/8fe27032
Branch: refs/heads/HAWQ-1228
Commit: 8fe27032f1faf786544a5109e854d319f6e6a1c4
Parents: 0574e75
Author: Oleksandr Diachenko <od...@pivotal.io>
Authored: Fri Jan 6 18:39:32 2017 -0800
Committer: Oleksandr Diachenko <od...@pivotal.io>
Committed: Fri Jan 6 18:39:32 2017 -0800
----------------------------------------------------------------------
.../org/apache/hawq/pxf/api/OutputFormat.java | 2 +-
.../plugins/hive/HiveColumnarSerdeResolver.java | 2 +-
.../pxf/plugins/hive/HiveMetadataFetcher.java | 2 +-
.../pxf/plugins/hive/HiveORCSerdeResolver.java | 22 --------------------
.../hawq/pxf/plugins/hive/HiveResolver.java | 3 ---
.../plugins/hive/HiveStringPassResolver.java | 2 +-
src/backend/catalog/external/externalmd.c | 2 +-
src/bin/gpfusion/gpbridgeapi.c | 4 ++--
src/include/access/hd_work_mgr.h | 2 ++
9 files changed, 9 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
index 82a747f..230f9ff 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
@@ -23,4 +23,4 @@ package org.apache.hawq.pxf.api;
/**
* PXF supported output formats: {@link #TEXT} and {@link #BINARY}
*/
-public enum OutputFormat {TEXT, BINARY, UNKNOWN}
+public enum OutputFormat {TEXT, BINARY}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
index 2bf39ff..8fd37e0 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
@@ -241,6 +241,6 @@ public class HiveColumnarSerdeResolver extends HiveResolver {
@Override
void parseDelimiterChar(InputData input) {
- delimiter = 1;
+ delimiter = 44; //,
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
index 3d04bc1..76e8e18 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
@@ -162,7 +162,7 @@ public class HiveMetadataFetcher extends MetadataFetcher {
}
private OutputFormat getOutputFormat(String inputFormat) {
- OutputFormat outputFormat = OutputFormat.UNKNOWN;
+ OutputFormat outputFormat = null;
try {
InputFormat<?, ?> fformat = HiveDataFragmenter.makeInputFormat(inputFormat, jobConf);
String profile = ProfileFactory.get(fformat);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
index 381c407..0c7a417 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
@@ -44,7 +44,6 @@ import java.util.*;
*/
public class HiveORCSerdeResolver extends HiveResolver {
private static final Log LOG = LogFactory.getLog(HiveORCSerdeResolver.class);
- private OrcSerde deserializer;
private HiveUtilities.PXF_HIVE_SERDES serdeType;
public HiveORCSerdeResolver(InputData input) throws Exception {
@@ -63,27 +62,6 @@ public class HiveORCSerdeResolver extends HiveResolver {
: input.getUserProperty("MAPKEY_DELIM");
}
- /**
- * getFields returns a singleton list of OneField item.
- * OneField item contains two fields: an integer representing the VARCHAR type and a Java
- * Object representing the field value.
- */
- //TODO: It's the same as in parent class
- @Override
- public List<OneField> getFields(OneRow onerow) throws Exception {
-
- Object tuple = deserializer.deserialize((Writable) onerow.getData());
- // Each Hive record is a Struct
- StructObjectInspector soi = (StructObjectInspector) deserializer.getObjectInspector();
- List<OneField> record = traverseStruct(tuple, soi, false);
-
- //Add partition fields if any
- record.addAll(getPartitionFields());
-
- return record;
-
- }
-
/*
* Get and init the deserializer for the records of this Hive data fragment.
* Suppress Warnings added because deserializer.initialize is an abstract function that is deprecated
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java
index 55d7205..b0dc2fb 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java
@@ -75,12 +75,9 @@ public class HiveResolver extends Plugin implements ReadResolver {
protected static final String COLLECTION_DELIM = ",";
protected String collectionDelim;
protected String mapkeyDelim;
- //private SerDe deserializer;
protected SerDe deserializer;
private List<OneField> partitionFields;
- //private String serdeClassName;
protected String serdeClassName;
- //private String propsString;
protected String propsString;
String partitionKeys;
protected char delimiter;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveStringPassResolver.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveStringPassResolver.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveStringPassResolver.java
index c7cfb36..8c91d47 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveStringPassResolver.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveStringPassResolver.java
@@ -93,6 +93,6 @@ public class HiveStringPassResolver extends HiveResolver {
}
void parseDelimiterChar(InputData input) {
- this.delimiter = 1;
+ delimiter = 44; //,
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/src/backend/catalog/external/externalmd.c
----------------------------------------------------------------------
diff --git a/src/backend/catalog/external/externalmd.c b/src/backend/catalog/external/externalmd.c
index ccdbdd6..c6cfb69 100644
--- a/src/backend/catalog/external/externalmd.c
+++ b/src/backend/catalog/external/externalmd.c
@@ -659,7 +659,7 @@ static Datum GetFormatOptionsForProfile(const List *outputFormats)
initStringInfo(&formatStr);
if (list_length(outputFormats) == 1 && strcmp(lfirst(list_head(outputFormats)),"TEXT") == 0)
{
- appendStringInfo(&formatStr, "delimiter '\x01' null '\N' escape '\'");
+ appendStringInfo(&formatStr, "delimiter '\x2C' null '\\N' escape '\\'");
} else {
appendStringInfo(&formatStr, "formatter 'pxfwritable_import'");
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/src/bin/gpfusion/gpbridgeapi.c
----------------------------------------------------------------------
diff --git a/src/bin/gpfusion/gpbridgeapi.c b/src/bin/gpfusion/gpbridgeapi.c
index c5c217c..358c858 100644
--- a/src/bin/gpfusion/gpbridgeapi.c
+++ b/src/bin/gpfusion/gpbridgeapi.c
@@ -564,7 +564,7 @@ void free_token_resources(PxfInputData *inputData)
static void assign_optimal_supported_profile(char *profile, char *fmttype, char **supportedProfile, char **supportedFormat)
{
- if (fmttype_is_text(*fmttype) && ((strcmp(profile, "HiveText") == 0) || (strcmp(profile, "HiveRc") == 0)))
+ if (fmttype_is_text(*fmttype) && ((strcmp(profile, HiveTextProfileName) == 0) || (strcmp(profile, HiveRCProfileName) == 0)))
{
*supportedFormat = "TEXT";
*supportedProfile = profile;
@@ -575,6 +575,6 @@ static void assign_optimal_supported_profile(char *profile, char *fmttype, char
} else
{
*supportedFormat = "GPDBWritable";
- *supportedProfile = "Hive";
+ *supportedProfile = HiveProfileName;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/src/include/access/hd_work_mgr.h
----------------------------------------------------------------------
diff --git a/src/include/access/hd_work_mgr.h b/src/include/access/hd_work_mgr.h
index cab8ca7..ea4c6ef 100644
--- a/src/include/access/hd_work_mgr.h
+++ b/src/include/access/hd_work_mgr.h
@@ -48,5 +48,7 @@ PxfFragmentStatsElem *get_pxf_fragments_statistics(char *uri, Relation rel);
List *get_pxf_item_metadata(char *profile, char *pattern, Oid dboid);
#define HiveProfileName "Hive"
+#define HiveTextProfileName "HiveText"
+#define HiveRCProfileName "HiveRC"
#endif /* HDWORKMGR_H */