You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by es...@apache.org on 2017/02/03 09:00:46 UTC
[44/50] [abbrv] incubator-hawq git commit: HAWQ-1228. Use profile
based on file format in HCatalog integration(HiveRC, HiveText profiles).
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
index b736bba..05ef001 100644
--- a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
+++ b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
@@ -23,10 +23,18 @@ package org.apache.hawq.pxf.plugins.hive.utilities;
import static org.junit.Assert.*;
import java.util.Arrays;
+import java.util.Collections;
import com.google.common.base.Joiner;
+
import org.apache.hawq.pxf.api.io.DataType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.ql.io.orc.OrcSerde;
+import org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe;
+import org.apache.hadoop.hive.serde2.*;
import org.junit.Test;
import org.apache.hawq.pxf.api.Metadata;
import org.apache.hawq.pxf.api.UnsupportedTypeException;
@@ -399,4 +407,49 @@ public class HiveUtilitiesTest {
assertEquals(errorMsg, e.getMessage());
}
}
+
+ @Test
+ public void createDeserializer() throws Exception {
+ SerDe serde = HiveUtilities.createDeserializer(HiveUtilities.PXF_HIVE_SERDES.ORC_SERDE, HiveUtilities.PXF_HIVE_SERDES.ORC_SERDE);
+ assertTrue(serde instanceof OrcSerde);
+
+ serde = HiveUtilities.createDeserializer(HiveUtilities.PXF_HIVE_SERDES.LAZY_BINARY_COLUMNAR_SERDE, HiveUtilities.PXF_HIVE_SERDES.LAZY_BINARY_COLUMNAR_SERDE);
+ assertTrue(serde instanceof org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe);
+
+ serde = HiveUtilities.createDeserializer(HiveUtilities.PXF_HIVE_SERDES.COLUMNAR_SERDE, HiveUtilities.PXF_HIVE_SERDES.COLUMNAR_SERDE);
+ assertTrue(serde instanceof org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe);
+
+ try {
+ serde = HiveUtilities.createDeserializer(HiveUtilities.PXF_HIVE_SERDES.COLUMNAR_SERDE, HiveUtilities.PXF_HIVE_SERDES.ORC_SERDE);
+ fail("shouldn't be able to create deserializer with not allowed serde");
+ } catch (UnsupportedTypeException e) {
+ assertTrue(e.getMessage().equals("Unsupported Hive Serde: " + HiveUtilities.PXF_HIVE_SERDES.COLUMNAR_SERDE.name()));
+ }
+ }
+
+ @Test
+ public void getDelimiterCode() {
+
+ //Default delimiter code should be 44(comma)
+ Integer delimiterCode = HiveUtilities.getDelimiterCode(null);
+ char defaultDelim = ',';
+ assertTrue(delimiterCode == (int) defaultDelim);
+
+ //Some serdes use FIELD_DELIM key
+ char expectedDelim = '%';
+ StorageDescriptor sd = new StorageDescriptor();
+ SerDeInfo si = new SerDeInfo();
+ si.setParameters(Collections.singletonMap(serdeConstants.FIELD_DELIM, String.valueOf(expectedDelim)));
+ sd.setSerdeInfo(si);
+ delimiterCode = HiveUtilities.getDelimiterCode(sd);
+ assertTrue(delimiterCode == (int) expectedDelim);
+
+ //Some serdes use SERIALIZATION_FORMAT key
+ sd = new StorageDescriptor();
+ si = new SerDeInfo();
+ si.setParameters(Collections.singletonMap(serdeConstants.SERIALIZATION_FORMAT, String.valueOf((int)expectedDelim)));
+ sd.setSerdeInfo(si);
+ delimiterCode = HiveUtilities.getDelimiterCode(sd);
+ assertTrue(delimiterCode == (int) expectedDelim);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/ProfileFactoryTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/ProfileFactoryTest.java b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/ProfileFactoryTest.java
new file mode 100644
index 0000000..d588d35
--- /dev/null
+++ b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/ProfileFactoryTest.java
@@ -0,0 +1,65 @@
+package org.apache.hawq.pxf.plugins.hive.utilities;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
+import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.SequenceFileInputFilter;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class ProfileFactoryTest {
+
+ @Test
+ public void get() throws Exception {
+
+ // For TextInputFormat when table has no complex types, HiveText profile should be used
+ String profileName = ProfileFactory.get(new TextInputFormat(), false);
+ assertEquals("HiveText", profileName);
+
+ // For TextInputFormat when table has complex types, Hive profile should be used, HiveText doesn't support complex types yet
+ profileName = ProfileFactory.get(new TextInputFormat(), true);
+ assertEquals("Hive", profileName);
+
+ // For RCFileInputFormat when table has complex types, HiveRC profile should be used
+ profileName = ProfileFactory.get(new RCFileInputFormat(), true);
+ assertEquals("HiveRC", profileName);
+
+ // For RCFileInputFormat when table has no complex types, HiveRC profile should be used
+ profileName = ProfileFactory.get(new RCFileInputFormat(), false);
+ assertEquals("HiveRC", profileName);
+
+ // For OrcInputFormat when table has complex types, HiveORC profile should be used
+ profileName = ProfileFactory.get(new OrcInputFormat(), true);
+ assertEquals("HiveORC", profileName);
+
+ // For OrcInputFormat when table has no complex types, HiveORC profile should be used
+ profileName = ProfileFactory.get(new OrcInputFormat(), false);
+ assertEquals("HiveORC", profileName);
+
+ // For other formats Hive profile should be used
+ profileName = ProfileFactory.get(new SequenceFileInputFilter(), false);
+ assertEquals("Hive", profileName);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/BridgeOutputBuilder.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/BridgeOutputBuilder.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/BridgeOutputBuilder.java
index d04a2f4..1c199d3 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/BridgeOutputBuilder.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/BridgeOutputBuilder.java
@@ -92,7 +92,7 @@ public class BridgeOutputBuilder {
void makeErrorRecord() {
int[] errSchema = { TEXT.getOID() };
- if (inputData.outputFormat() != OutputFormat.BINARY) {
+ if (inputData.outputFormat() != OutputFormat.GPDBWritable) {
return;
}
@@ -109,7 +109,7 @@ public class BridgeOutputBuilder {
* @throws Exception if the output format is not binary
*/
public Writable getErrorOutput(Exception ex) throws Exception {
- if (inputData.outputFormat() == OutputFormat.BINARY) {
+ if (inputData.outputFormat() == OutputFormat.GPDBWritable) {
errorRecord.setString(0, ex.getMessage());
return errorRecord;
} else {
@@ -126,7 +126,7 @@ public class BridgeOutputBuilder {
*/
public LinkedList<Writable> makeOutput(List<OneField> recFields)
throws BadRecordException {
- if (output == null && inputData.outputFormat() == OutputFormat.BINARY) {
+ if (output == null && inputData.outputFormat() == OutputFormat.GPDBWritable) {
makeGPDBWritableOutput();
}
@@ -174,7 +174,7 @@ public class BridgeOutputBuilder {
* @throws BadRecordException if building the output record failed
*/
void fillOutputRecord(List<OneField> recFields) throws BadRecordException {
- if (inputData.outputFormat() == OutputFormat.BINARY) {
+ if (inputData.outputFormat() == OutputFormat.GPDBWritable) {
fillGPDBWritable(recFields);
} else {
fillText(recFields);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
index 8225ec5..d2b0b5c 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
@@ -86,7 +86,8 @@ public class MetadataResponseFormatter {
result.append("Field #").append(++i).append(": [")
.append("Name: ").append(field.getName())
.append(", Type: ").append(field.getType().getTypeName())
- .append(", Source type: ").append(field.getSourceType()).append("] ");
+ .append(", Source type: ").append(field.getSourceType())
+ .append(", Source type is complex: ").append(field.isComplexType()).append("] ");
}
}
LOG.debug(result);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/ProfileFactory.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/ProfileFactory.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/ProfileFactory.java
deleted file mode 100644
index fc5ed0f..0000000
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/ProfileFactory.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package org.apache.hawq.pxf.service;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
-import org.apache.hadoop.mapred.InputFormat;
-
-public class ProfileFactory {
-
- private static final String HIVE_TEXT_PROFILE = "HiveText";
- private static final String HIVE_RC_PROFILE = "HiveRC";
- private static final String HIVE_ORC_PROFILE = "HiveORC";
-
- public static String get(InputFormat inputFormat) throws Exception {
- String profileName = null;
- // TODO: Uncomment in process of HAWQ-1228 implementation
- //if (inputFormat instanceof TextInputFormat) {
- // profileName = HIVE_TEXT_PROFILE;
- //} else if (inputFormat instanceof RCFileInputFormat) {
- // profileName = HIVE_RC_PROFILE;
- /*} else */if (inputFormat instanceof OrcInputFormat) {
- profileName = HIVE_ORC_PROFILE;
- }
-
- return profileName;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/MetadataResource.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/MetadataResource.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/MetadataResource.java
index 3f85bb8..5263ea2 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/MetadataResource.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/MetadataResource.java
@@ -50,7 +50,7 @@ import org.apache.hawq.pxf.service.utilities.SecuredHDFS;
* Class enhances the API of the WEBHDFS REST server. Returns the metadata of a
* given hcatalog table. <br>
* Example for querying API FRAGMENTER from a web client:<br>
- * <code>curl -i "http://localhost:51200/pxf/{version}/Metadata/getTableMetadata?table=t1"</code>
+ * <code>curl -i "http://localhost:51200/pxf/{version}/Metadata/getMetadata?profile=PROFILE_NAME&pattern=OBJECT_PATTERN"</code>
* <br>
* /pxf/ is made part of the path when there is a webapp by that name in tomcat.
*/
@@ -69,7 +69,12 @@ public class MetadataResource extends RestResource {
* Response Examples:<br>
* For a table <code>default.t1</code> with 2 fields (a int, b float) will
* be returned as:
- * <code>{"PXFMetadata":[{"item":{"path":"default","name":"t1"},"fields":[{"name":"a","type":"int"},{"name":"b","type":"float"}]}]}</code>
+ * <code>{"PXFMetadata":[{"item":{"path":"default","name":"t1"},
+ * "fields":[{"name":"a","type":"int4","sourceType":"int","complexType":false},
+ * {"name":"b","type":"float8","sourceType":"double","complexType":false}],
+ * "outputFormats":["TEXT"],
+ * "outputParameters":{"DELIMITER":"1"}}]}
+ * </code>
*
* @param servletContext servlet context
* @param headers http headers
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/VersionResource.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/VersionResource.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/VersionResource.java
index db9743e..c9f4d20 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/VersionResource.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/VersionResource.java
@@ -31,7 +31,7 @@ import org.apache.commons.logging.LogFactory;
/**
* PXF protocol version. Any call to PXF resources should include the current
- * version e.g. {@code ...pxf/v14/Bridge}
+ * version e.g. {@code ...pxf/v15/Bridge}
*/
class Version {
/**
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
index a0e63ce..dc2a110 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
@@ -71,7 +71,7 @@ public class ProtocolData extends InputData {
filterString = getProperty("FILTER");
}
- parseFormat(getProperty("FORMAT"));
+ outputFormat = OutputFormat.valueOf(getProperty("FORMAT"));
host = getProperty("URL-HOST");
port = getIntProperty("URL-PORT");
@@ -359,26 +359,6 @@ public class ProtocolData extends InputData {
+ threadSafeStr + "'." + " Usage: [TRUE|FALSE]");
}
- /**
- * Sets the format type based on the input string. Allowed values are:
- * "TEXT", "GPDBWritable".
- *
- * @param formatString format string
- */
- protected void parseFormat(String formatString) {
- switch (formatString) {
- case "TEXT":
- outputFormat = OutputFormat.TEXT;
- break;
- case "GPDBWritable":
- outputFormat = OutputFormat.BINARY;
- break;
- default:
- throw new IllegalArgumentException(
- "Wrong value for greenplum.format " + formatString);
- }
- }
-
/*
* Sets the tuple description for the record
* Attribute Projection information is optional
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml b/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
index 1edb6d5..d36f54b 100644
--- a/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
+++ b/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
@@ -43,12 +43,16 @@ under the License.
</profile>
<profile>
<name>Hive</name>
- <description>This profile is suitable for using when connecting to Hive</description>
+ <description>
+ This profile is suitable for using when connecting to Hive.
+ Supports GPDBWritable output format, as specified in FORMAT header parameter.
+ </description>
<plugins>
<fragmenter>org.apache.hawq.pxf.plugins.hive.HiveDataFragmenter</fragmenter>
<accessor>org.apache.hawq.pxf.plugins.hive.HiveAccessor</accessor>
<resolver>org.apache.hawq.pxf.plugins.hive.HiveResolver</resolver>
<metadata>org.apache.hawq.pxf.plugins.hive.HiveMetadataFetcher</metadata>
+ <outputFormat>org.apache.hawq.pxf.service.io.GPDBWritable</outputFormat>
</plugins>
</profile>
<profile>
@@ -57,12 +61,15 @@ under the License.
and serialized with either the ColumnarSerDe or the LazyBinaryColumnarSerDe.
It is much faster than the general purpose Hive profile.
DELIMITER parameter is mandatory.
+ Supports both GPDBWritable and TEXT output formats, as specified in FORMAT header parameter.
+ Primary optimized for TEXT output format.
</description>
<plugins>
<fragmenter>org.apache.hawq.pxf.plugins.hive.HiveInputFormatFragmenter</fragmenter>
<accessor>org.apache.hawq.pxf.plugins.hive.HiveRCFileAccessor</accessor>
<resolver>org.apache.hawq.pxf.plugins.hive.HiveColumnarSerdeResolver</resolver>
<metadata>org.apache.hawq.pxf.plugins.hive.HiveMetadataFetcher</metadata>
+ <outputFormat>org.apache.hawq.pxf.service.io.Text</outputFormat>
</plugins>
</profile>
<profile>
@@ -70,12 +77,15 @@ under the License.
<description>This profile is suitable only for Hive tables stored as Text files.
It is much faster than the general purpose Hive profile.
DELIMITER parameter is mandatory.
+ Supports both GPDBWritable and TEXT output formats, as specified in FORMAT header parameter.
+ Primary optimized for TEXT output format.
</description>
<plugins>
<fragmenter>org.apache.hawq.pxf.plugins.hive.HiveInputFormatFragmenter</fragmenter>
<accessor>org.apache.hawq.pxf.plugins.hive.HiveLineBreakAccessor</accessor>
<resolver>org.apache.hawq.pxf.plugins.hive.HiveStringPassResolver</resolver>
<metadata>org.apache.hawq.pxf.plugins.hive.HiveMetadataFetcher</metadata>
+ <outputFormat>org.apache.hawq.pxf.service.io.Text</outputFormat>
</plugins>
</profile>
<profile>
@@ -83,12 +93,14 @@ under the License.
<description>This profile is suitable only for Hive tables stored in ORC files
and serialized with either the ColumnarSerDe or the LazyBinaryColumnarSerDe.
It is much faster than the general purpose Hive profile.
+ Supports GPDBWritable output format, as specified in FORMAT header parameter.
</description>
<plugins>
<fragmenter>org.apache.hawq.pxf.plugins.hive.HiveInputFormatFragmenter</fragmenter>
<accessor>org.apache.hawq.pxf.plugins.hive.HiveORCAccessor</accessor>
<resolver>org.apache.hawq.pxf.plugins.hive.HiveORCSerdeResolver</resolver>
<metadata>org.apache.hawq.pxf.plugins.hive.HiveMetadataFetcher</metadata>
+ <outputFormat>org.apache.hawq.pxf.service.io.GPDBWritable</outputFormat>
</plugins>
</profile>
<profile>
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/MetadataResponseFormatterTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/MetadataResponseFormatterTest.java b/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/MetadataResponseFormatterTest.java
index 21bf423..546b42d 100644
--- a/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/MetadataResponseFormatterTest.java
+++ b/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/MetadataResponseFormatterTest.java
@@ -57,7 +57,7 @@ public class MetadataResponseFormatterTest {
response = MetadataResponseFormatter.formatResponse(metadataList, "path.file");
StringBuilder expected = new StringBuilder("{\"PXFMetadata\":[{");
expected.append("\"item\":{\"path\":\"default\",\"name\":\"table1\"},")
- .append("\"fields\":[{\"name\":\"field1\",\"type\":\"int8\",\"sourceType\":\"bigint\"},{\"name\":\"field2\",\"type\":\"text\",\"sourceType\":\"string\"}]}]}");
+ .append("\"fields\":[{\"name\":\"field1\",\"type\":\"int8\",\"sourceType\":\"bigint\",\"complexType\":false},{\"name\":\"field2\",\"type\":\"text\",\"sourceType\":\"string\",\"complexType\":false}]}]}");
assertEquals(expected.toString(), convertResponseToString(response));
}
@@ -75,7 +75,7 @@ public class MetadataResponseFormatterTest {
response = MetadataResponseFormatter.formatResponse(metadataList, "path.file");
StringBuilder expected = new StringBuilder("{\"PXFMetadata\":[{");
expected.append("\"item\":{\"path\":\"default\",\"name\":\"table1\"},")
- .append("\"fields\":[{\"name\":\"field1\",\"type\":\"int8\",\"sourceType\":\"bigint\"},{\"name\":\"field2\",\"type\":\"text\",\"sourceType\":\"string\"}]}]}");
+ .append("\"fields\":[{\"name\":\"field1\",\"type\":\"int8\",\"sourceType\":\"bigint\",\"complexType\":false},{\"name\":\"field2\",\"type\":\"text\",\"sourceType\":\"string\",\"complexType\":false}]}]}");
assertEquals(expected.toString(), convertResponseToString(response));
}
@@ -97,9 +97,9 @@ public class MetadataResponseFormatterTest {
StringBuilder expected = new StringBuilder("{\"PXFMetadata\":[{");
expected.append("\"item\":{\"path\":\"default\",\"name\":\"table1\"},")
.append("\"fields\":[")
- .append("{\"name\":\"field1\",\"type\":\"int8\",\"sourceType\":\"bigint\"},")
- .append("{\"name\":\"field2\",\"type\":\"numeric\",\"sourceType\":\"decimal\",\"modifiers\":[\"1349\",\"1789\"]},")
- .append("{\"name\":\"field3\",\"type\":\"bpchar\",\"sourceType\":\"char\",\"modifiers\":[\"50\"]}")
+ .append("{\"name\":\"field1\",\"type\":\"int8\",\"sourceType\":\"bigint\",\"complexType\":false},")
+ .append("{\"name\":\"field2\",\"type\":\"numeric\",\"sourceType\":\"decimal\",\"modifiers\":[\"1349\",\"1789\"],\"complexType\":false},")
+ .append("{\"name\":\"field3\",\"type\":\"bpchar\",\"sourceType\":\"char\",\"modifiers\":[\"50\"],\"complexType\":false}")
.append("]}]}");
assertEquals(expected.toString(), convertResponseToString(response));
@@ -118,7 +118,7 @@ public class MetadataResponseFormatterTest {
StringBuilder expected = new StringBuilder("{\"PXFMetadata\":[{");
expected.append("\"item\":{\"path\":\"default\",\"name\":\"table1\"},")
.append("\"fields\":[")
- .append("{\"name\":\"field1\",\"type\":\"float8\",\"sourceType\":\"double\"}")
+ .append("{\"name\":\"field1\",\"type\":\"float8\",\"sourceType\":\"double\",\"complexType\":false}")
.append("]}]}");
assertEquals(expected.toString(), convertResponseToString(response));
@@ -199,7 +199,7 @@ public class MetadataResponseFormatterTest {
expected.append(",");
}
expected.append("{\"item\":{\"path\":\"default\",\"name\":\"table").append(i).append("\"},");
- expected.append("\"fields\":[{\"name\":\"field1\",\"type\":\"int8\",\"sourceType\":\"bigint\"},{\"name\":\"field2\",\"type\":\"text\",\"sourceType\":\"string\"}]}");
+ expected.append("\"fields\":[{\"name\":\"field1\",\"type\":\"int8\",\"sourceType\":\"bigint\",\"complexType\":false},{\"name\":\"field2\",\"type\":\"text\",\"sourceType\":\"string\",\"complexType\":false}]}");
}
expected.append("]}");
@@ -226,7 +226,7 @@ public class MetadataResponseFormatterTest {
expected.append(",");
}
expected.append("{\"item\":{\"path\":\"default").append(i).append("\",\"name\":\"table").append(i).append("\"},");
- expected.append("\"fields\":[{\"name\":\"field1\",\"type\":\"int8\",\"sourceType\":\"bigint\"},{\"name\":\"field2\",\"type\":\"text\",\"sourceType\":\"string\"}]}");
+ expected.append("\"fields\":[{\"name\":\"field1\",\"type\":\"int8\",\"sourceType\":\"bigint\",\"complexType\":false},{\"name\":\"field2\",\"type\":\"text\",\"sourceType\":\"string\",\"complexType\":false}]}");
}
expected.append("]}");
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/src/backend/access/external/fileam.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/fileam.c b/src/backend/access/external/fileam.c
index 70a115a..d16d516 100644
--- a/src/backend/access/external/fileam.c
+++ b/src/backend/access/external/fileam.c
@@ -461,8 +461,11 @@ external_stopscan(FileScanDesc scan)
ExternalSelectDesc
external_getnext_init(PlanState *state) {
ExternalSelectDesc desc = (ExternalSelectDesc) palloc0(sizeof(ExternalSelectDescData));
+
if (state != NULL)
+ {
desc->projInfo = state->ps_ProjInfo;
+ }
return desc;
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/src/backend/access/external/pxfheaders.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/pxfheaders.c b/src/backend/access/external/pxfheaders.c
index 2381044..a60dc48 100644
--- a/src/backend/access/external/pxfheaders.c
+++ b/src/backend/access/external/pxfheaders.c
@@ -56,7 +56,7 @@ void build_http_header(PxfInputData *input)
/* format */
ExtTableEntry *exttbl = GetExtTableEntry(rel->rd_id);
/* pxf treats CSV as TEXT */
- char* format = (fmttype_is_text(exttbl->fmtcode) || fmttype_is_csv(exttbl->fmtcode)) ? "TEXT":"GPDBWritable";
+ char* format = get_format_name(exttbl->fmtcode);
churl_headers_append(headers, "X-GP-FORMAT", format);
/* Record fields - name and type of each field */
@@ -338,3 +338,22 @@ static void add_remote_credentials(CHURL_HEADERS headers)
if (pxf_remote_service_secret != NULL)
churl_headers_append(headers, "X-GP-REMOTE-PASS", pxf_remote_service_secret);
}
+
+char* get_format_name(char fmtcode)
+{
+ char *formatName = NULL;
+
+ if (fmttype_is_text(fmtcode) || fmttype_is_csv(fmtcode))
+ {
+ formatName = TextFormatName;
+ } else if (fmttype_is_custom(fmtcode))
+ {
+ formatName = GpdbWritableFormatName;
+ } else {
+ ereport(ERROR,
+ (errcode(ERRCODE_INTERNAL_ERROR),
+ errmsg("Unable to get format name for format code: %c", fmtcode)));
+ }
+
+ return formatName;
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/src/backend/access/external/test/pxfheaders_test.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/test/pxfheaders_test.c b/src/backend/access/external/test/pxfheaders_test.c
index 454ecdc..851483d 100644
--- a/src/backend/access/external/test/pxfheaders_test.c
+++ b/src/backend/access/external/test/pxfheaders_test.c
@@ -125,6 +125,22 @@ test__build_http_header__remote_credentials_are_not_null(void **state)
build_http_header(input_data);
}
+void
+test__get_format_name(void **state)
+{
+ char fmtcode = 't';
+ char *formatName = get_format_name(fmtcode);
+ assert_string_equal(TextFormatName, formatName);
+
+ fmtcode = 'c';
+ formatName = get_format_name(fmtcode);
+ assert_string_equal(TextFormatName, formatName);
+
+ fmtcode = 'b';
+ formatName = get_format_name(fmtcode);
+ assert_string_equal(GpdbWritableFormatName, formatName);
+}
+
/*
* Add an expect clause on a churl_headers_append with given
* key and value
@@ -244,6 +260,8 @@ main(int argc, char* argv[])
unit_test_setup_teardown(test__build_http_header__remote_login_is_not_null,
common_setup, common_teardown),
unit_test_setup_teardown(test__build_http_header__remote_credentials_are_not_null,
+ common_setup, common_teardown),
+ unit_test_setup_teardown(test__get_format_name,
common_setup, common_teardown)
};
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/src/backend/catalog/external/externalmd.c
----------------------------------------------------------------------
diff --git a/src/backend/catalog/external/externalmd.c b/src/backend/catalog/external/externalmd.c
index 0e39d25..3a62f17 100644
--- a/src/backend/catalog/external/externalmd.c
+++ b/src/backend/catalog/external/externalmd.c
@@ -57,6 +57,9 @@ static void LoadDistributionPolicy(Oid relid, PxfItem *pxfItem);
static void LoadExtTable(Oid relid, PxfItem *pxfItem);
static void LoadColumns(Oid relid, List *columns);
static int ComputeTypeMod(Oid typeOid, const char *colname, int *typemod, int nTypeMod);
+static Datum GetFormatTypeForProfile(const List *outputFormats);
+static Datum GetFormatOptionsForProfile(const List *outputFormats, int delimiter);
+static Datum GetLocationForFormat(char *profile, List *outputFormats, char *pxf_service_address, char *path, char *name, int delimiter);
const int maxNumTypeModifiers = 2;
/*
@@ -124,11 +127,33 @@ static PxfItem *ParsePxfItem(struct json_object *pxfMD, char* profile)
ereport(ERROR,
(errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
errmsg("Could not parse PXF item, expected not null value for attribute \"name\"")));
-
pxfItem->profile = profile;
pxfItem->path = pstrdup(json_object_get_string(itemPath));
pxfItem->name = pstrdup(json_object_get_string(itemName));
-
+
+ /* parse output formats */
+ struct json_object *jsonOutputFormats = json_object_object_get(pxfMD, "outputFormats");
+
+ if (NULL != jsonOutputFormats)
+ {
+ const int numOutputFormats = json_object_array_length(jsonOutputFormats);
+ for (int i = 0; i < numOutputFormats; i++)
+ {
+ PxfField *pxfField = palloc0(sizeof(PxfField));
+ struct json_object *jsonOutputFormat = json_object_array_get_idx(jsonOutputFormats, i);
+ char *outupFormat = pstrdup(json_object_get_string(jsonOutputFormat));
+ pxfItem->outputFormats = lappend(pxfItem->outputFormats, outupFormat);
+ }
+ }
+
+ /* parse delimiter */
+ struct json_object *jsonOutputParameters = json_object_object_get(pxfMD, "outputParameters");
+ if (NULL != jsonOutputParameters)
+ {
+ struct json_object *outputParameterDelimiter = json_object_object_get(jsonOutputParameters, "DELIMITER");
+ pxfItem->delimiter = atoi(pstrdup(json_object_get_string(outputParameterDelimiter)));
+ }
+
elog(DEBUG1, "Parsed item %s, namespace %s", itemName, itemPath);
/* parse columns */
@@ -445,36 +470,10 @@ static void LoadExtTable(Oid relid, PxfItem *pxfItem)
values[i] = (Datum) 0;
}
- /* location - should be an array of text with one element:
- * pxf://<ip:port/namaservice>/<hive db>.<hive table>?Profile=Hive */
- StringInfoData locationStr;
- initStringInfo(&locationStr);
- appendStringInfo(&locationStr, "pxf://%s/%s.%s?Profile=%s",
- pxf_service_address, pxfItem->path, pxfItem->name, pxfItem->profile);
- Size len = VARHDRSZ + locationStr.len;
- /* +1 leaves room for sprintf's trailing null */
- text *t = (text *) palloc(len + 1);
- SET_VARSIZE(t, len);
- sprintf((char *) VARDATA(t), "%s", locationStr.data);
- ArrayBuildState *astate = NULL;
- astate = accumArrayResult(astate, PointerGetDatum(t),
- false, TEXTOID,
- CurrentMemoryContext);
- pfree(locationStr.data);
- Assert(NULL != astate);
- Datum location = makeArrayResult(astate, CurrentMemoryContext);
-
- /* format options - should be "formatter 'pxfwritable_import'" */
- StringInfoData formatStr;
- initStringInfo(&formatStr);
- appendStringInfo(&formatStr, "formatter 'pxfwritable_import'");
- Datum format_opts = DirectFunctionCall1(textin, CStringGetDatum(formatStr.data));
- pfree(formatStr.data);
-
values[Anum_pg_exttable_reloid - 1] = ObjectIdGetDatum(relid);
- values[Anum_pg_exttable_location - 1] = location;
- values[Anum_pg_exttable_fmttype - 1] = CharGetDatum('b' /* binary */);
- values[Anum_pg_exttable_fmtopts - 1] = format_opts;
+ values[Anum_pg_exttable_location - 1] = GetLocationForFormat(pxfItem->profile, pxfItem->outputFormats, pxf_service_address, pxfItem->path, pxfItem->name, pxfItem->delimiter);
+ values[Anum_pg_exttable_fmttype - 1] = GetFormatTypeForProfile(pxfItem->outputFormats);
+ values[Anum_pg_exttable_fmtopts - 1] = GetFormatOptionsForProfile(pxfItem->outputFormats, pxfItem->delimiter);
nulls[Anum_pg_exttable_command - 1] = true;
nulls[Anum_pg_exttable_rejectlimit - 1] = true;
nulls[Anum_pg_exttable_rejectlimittype - 1] = true;
@@ -631,3 +630,79 @@ static int ComputeTypeMod(Oid typeOid, const char *colname, int *typemod, int nT
return VARHDRSZ + result;
}
+static Datum GetFormatTypeForProfile(const List *outputFormats)
+{
+
+ /* if table is homogeneous and output format is text - use text*/
+ if (list_length(outputFormats) == 1 && strcmp(lfirst(list_head(outputFormats)), TextFormatName) == 0)
+ {
+ return CharGetDatum(TextFormatType);
+ } else
+ {
+ return CharGetDatum(CustomFormatType);
+ }
+}
+
+static Datum GetFormatOptionsForProfile(const List *outputFormats, int delimiter)
+{
+ StringInfoData formatStr;
+ initStringInfo(&formatStr);
+
+ /* "delimiter 'delimiter' null '\\N' escape '\\'"*/
+ char formatArr[35] = { 0x64, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65,
+ 0x72, 0x20, 0x27, delimiter, 0x27, 0x20, 0x6e, 0x75, 0x6c, 0x6c,
+ 0x20, 0x27, 0x5c, 0x4e, 0x27, 0x20, 0x65, 0x73, 0x63, 0x61, 0x70,
+ 0x65, 0x20, 0x27, 0x5c, 0x27, 0x00 };
+
+ if (list_length(outputFormats) == 1 && strcmp(lfirst(list_head(outputFormats)),TextFormatName) == 0)
+ {
+ appendStringInfo(&formatStr, "%s", formatArr);
+ } else {
+ appendStringInfo(&formatStr, "formatter 'pxfwritable_import'");
+ }
+ Datum format_opts = DirectFunctionCall1(textin, CStringGetDatum(formatStr.data));
+ pfree(formatStr.data);
+ return format_opts;
+}
+
+/* location - should be an array of text with one element:
+ * pxf://<ip:port/namaservice>/<path>.<name>?Profile=profileName&delimiter=delimiterCode */
+static Datum GetLocationForFormat(char *profile, List *outputFormats, char *pxf_service_address, char *path, char *name, int delimiter)
+{
+ StringInfoData locationStr;
+ initStringInfo(&locationStr);
+ appendStringInfo(&locationStr, "pxf://%s/%s.%s?Profile=%s", pxf_service_address, path, name, profile);
+ bool hasTextOutputFormat = false;
+ ListCell *lc = NULL;
+ foreach (lc, outputFormats)
+ {
+ char *outputFormat = (char *) lfirst(lc);
+ if (strcmp(outputFormat, TextFormatName) == 0)
+ {
+ hasTextOutputFormat = true;
+ break;
+ }
+ }
+ if (delimiter)
+ {
+ appendStringInfo(&locationStr, "&delimiter=%cx%02x", '\\', delimiter);
+ } else if (hasTextOutputFormat)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
+ errmsg("delimiter attribute is mandatory for output format \"TEXT\"")));
+ }
+ Size len = VARHDRSZ + locationStr.len;
+ /* +1 leaves room for sprintf's trailing null */
+ text *t = (text *) palloc(len + 1);
+ SET_VARSIZE(t, len);
+ sprintf((char *) VARDATA(t), "%s", locationStr.data);
+ ArrayBuildState *astate = NULL;
+ astate = accumArrayResult(astate, PointerGetDatum(t),
+ false, TEXTOID,
+ CurrentMemoryContext);
+ pfree(locationStr.data);
+ Assert(NULL != astate);
+ Datum location = makeArrayResult(astate, CurrentMemoryContext);
+ return location;
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/src/bin/gpfusion/gpbridgeapi.c
----------------------------------------------------------------------
diff --git a/src/bin/gpfusion/gpbridgeapi.c b/src/bin/gpfusion/gpbridgeapi.c
index b524df8..cf4dd84 100644
--- a/src/bin/gpfusion/gpbridgeapi.c
+++ b/src/bin/gpfusion/gpbridgeapi.c
@@ -68,7 +68,6 @@ void free_token_resources(PxfInputData *inputData);
Datum gpbridge_import(PG_FUNCTION_ARGS)
{
gpbridge_check_inside_extproto(fcinfo, "gpbridge_import");
-// ExternalSelectDesc desc = EXTPROTOCOL_GET_SELECTDESC(fcinfo);
if (gpbridge_last_call(fcinfo))
PG_RETURN_INT32(gpbridge_cleanup(fcinfo));
@@ -226,14 +225,17 @@ void set_current_fragment_headers(gphadoop_context* context)
churl_headers_remove(context->churl_headers, "X-GP-FRAGMENT-USER-DATA", true);
}
- /* if current fragment has optimal profile set it*/
if (frag_data->profile)
{
+ /* if current fragment has optimal profile set it*/
churl_headers_override(context->churl_headers, "X-GP-PROFILE", frag_data->profile);
+ elog(DEBUG2, "pxf: set_current_fragment_headers: using profile: %s", frag_data->profile);
+
} else if (context->gphd_uri->profile)
{
/* if current fragment doesn't have any optimal profile, set to use profile from url */
churl_headers_override(context->churl_headers, "X-GP-PROFILE", context->gphd_uri->profile);
+ elog(DEBUG2, "pxf: set_current_fragment_headers: using profile: %s", context->gphd_uri->profile);
}
/* if there is no profile passed in url, we expect to have accessor+fragmenter+resolver so no action needed by this point */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/src/include/access/hd_work_mgr.h
----------------------------------------------------------------------
diff --git a/src/include/access/hd_work_mgr.h b/src/include/access/hd_work_mgr.h
index cab8ca7..ea4c6ef 100644
--- a/src/include/access/hd_work_mgr.h
+++ b/src/include/access/hd_work_mgr.h
@@ -48,5 +48,7 @@ PxfFragmentStatsElem *get_pxf_fragments_statistics(char *uri, Relation rel);
List *get_pxf_item_metadata(char *profile, char *pattern, Oid dboid);
#define HiveProfileName "Hive"
+#define HiveTextProfileName "HiveText"
+#define HiveRCProfileName "HiveRC"
#endif /* HDWORKMGR_H */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/src/include/access/pxfheaders.h
----------------------------------------------------------------------
diff --git a/src/include/access/pxfheaders.h b/src/include/access/pxfheaders.h
index 410a077..f4adc6c 100644
--- a/src/include/access/pxfheaders.h
+++ b/src/include/access/pxfheaders.h
@@ -49,5 +49,6 @@ typedef struct sPxfInputData
} PxfInputData;
void build_http_header(PxfInputData *input);
+char* get_format_name(char fmtcode);
#endif
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/src/include/access/pxfuriparser.h
----------------------------------------------------------------------
diff --git a/src/include/access/pxfuriparser.h b/src/include/access/pxfuriparser.h
index ac614c5..77ad8b0 100644
--- a/src/include/access/pxfuriparser.h
+++ b/src/include/access/pxfuriparser.h
@@ -30,7 +30,7 @@
* All PXF's resources are under /PXF_SERVICE_PREFIX/PXF_VERSION/...
*/
#define PXF_SERVICE_PREFIX "pxf"
-#define PXF_VERSION "v14" /* PXF version */
+#define PXF_VERSION "v15" /* PXF version */
/*
* FragmentData - describes a single Hadoop file split / HBase table region
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/src/include/catalog/external/itemmd.h
----------------------------------------------------------------------
diff --git a/src/include/catalog/external/itemmd.h b/src/include/catalog/external/itemmd.h
index e6dad63..d9f8721 100644
--- a/src/include/catalog/external/itemmd.h
+++ b/src/include/catalog/external/itemmd.h
@@ -67,6 +67,11 @@ typedef struct PxfItem
/* fields */
List *fields;
+
+ /* output formats*/
+ List *outputFormats;
+
+ int delimiter;
} PxfItem;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/src/include/catalog/pg_exttable.h
----------------------------------------------------------------------
diff --git a/src/include/catalog/pg_exttable.h b/src/include/catalog/pg_exttable.h
index 3a0fadd..3256bb9 100644
--- a/src/include/catalog/pg_exttable.h
+++ b/src/include/catalog/pg_exttable.h
@@ -164,8 +164,16 @@ GetExtTableEntry(Oid relid);
extern void
RemoveExtTableEntry(Oid relid);
-#define fmttype_is_custom(c) (c == 'b')
-#define fmttype_is_text(c) (c == 't')
-#define fmttype_is_csv(c) (c == 'c')
+#define CustomFormatType 'b'
+#define TextFormatType 't'
+#define CsvFormatType 'c'
+
+/* PXF formats*/
+#define GpdbWritableFormatName "GPDBWritable"
+#define TextFormatName "TEXT"
+
+#define fmttype_is_custom(c) (c == CustomFormatType)
+#define fmttype_is_text(c) (c == TextFormatType)
+#define fmttype_is_csv(c) (c == CsvFormatType)
#endif /* PG_EXTTABLE_H */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/src/test/regress/data/hcatalog/single_table.json
----------------------------------------------------------------------
diff --git a/src/test/regress/data/hcatalog/single_table.json b/src/test/regress/data/hcatalog/single_table.json
index b571e5d..53cce73 100644
--- a/src/test/regress/data/hcatalog/single_table.json
+++ b/src/test/regress/data/hcatalog/single_table.json
@@ -1 +1 @@
-{"PXFMetadata":[{"item":{"path":"default","name":"mytable"},"fields":[{"name":"s1","type":"text","sourceType":"string"},{"name":"s2","type":"text","sourceType":"string"},{"name":"n1","type":"int4","sourceType":"int"},{"name":"d1","type":"float8","sourceType":"double"},{"name":"dc1","type":"numeric","modifiers":["38","18"],"sourceType":"decimal"},{"name":"tm","type":"timestamp","sourceType":"timestamp"},{"name":"f","type":"float4","sourceType":"float"},{"name":"bg","type":"int8","sourceType":"bigint"},{"name":"b","type":"bool","sourceType":"boolean"},{"name":"tn","type":"int2","sourceType":"tinyint"},{"name":"sml","type":"int2","sourceType":"tinyint"},{"name":"dt","type":"date","sourceType":"date"},{"name":"vc1","type":"varchar","modifiers":["5"],"sourceType":"varchar"},{"name":"c1","type":"bpchar","modifiers":["3"],"sourceType":"char"},{"name":"bin","type":"bytea","sourceType":"binary"}]}]}
+{"PXFMetadata":[{"item":{"path":"default","name":"mytable"},"fields":[{"name":"s1","type":"text","sourceType":"string"},{"name":"s2","type":"text","sourceType":"string"},{"name":"n1","type":"int4","sourceType":"int"},{"name":"d1","type":"float8","sourceType":"double"},{"name":"dc1","type":"numeric","modifiers":["38","18"],"sourceType":"decimal"},{"name":"tm","type":"timestamp","sourceType":"timestamp"},{"name":"f","type":"float4","sourceType":"float"},{"name":"bg","type":"int8","sourceType":"bigint"},{"name":"b","type":"bool","sourceType":"boolean"},{"name":"tn","type":"int2","sourceType":"tinyint"},{"name":"sml","type":"int2","sourceType":"tinyint"},{"name":"dt","type":"date","sourceType":"date"},{"name":"vc1","type":"varchar","modifiers":["5"],"sourceType":"varchar"},{"name":"c1","type":"bpchar","modifiers":["3"],"sourceType":"char"},{"name":"bin","type":"bytea","sourceType":"binary"}],"outputFormats":["GPDBWritable"],"outputParameters":{"DELIMITER":"1"}}]}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/src/test/regress/data/hcatalog/single_table_text.json
----------------------------------------------------------------------
diff --git a/src/test/regress/data/hcatalog/single_table_text.json b/src/test/regress/data/hcatalog/single_table_text.json
new file mode 100644
index 0000000..dbc301a
--- /dev/null
+++ b/src/test/regress/data/hcatalog/single_table_text.json
@@ -0,0 +1 @@
+{"PXFMetadata":[{"item":{"path":"default","name":"mytable"},"fields":[{"name":"s1","type":"text","sourceType":"string"},{"name":"s2","type":"text","sourceType":"string"},{"name":"n1","type":"int4","sourceType":"int"},{"name":"d1","type":"float8","sourceType":"double"},{"name":"dc1","type":"numeric","modifiers":["38","18"],"sourceType":"decimal"},{"name":"tm","type":"timestamp","sourceType":"timestamp"},{"name":"f","type":"float4","sourceType":"float"},{"name":"bg","type":"int8","sourceType":"bigint"},{"name":"b","type":"bool","sourceType":"boolean"},{"name":"tn","type":"int2","sourceType":"tinyint"},{"name":"sml","type":"int2","sourceType":"tinyint"},{"name":"dt","type":"date","sourceType":"date"},{"name":"vc1","type":"varchar","modifiers":["5"],"sourceType":"varchar"},{"name":"c1","type":"bpchar","modifiers":["3"],"sourceType":"char"},{"name":"bin","type":"bytea","sourceType":"binary"}],"outputFormats":["TEXT"],"outputParameters":{"DELIMITER":"42"}}]}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/src/test/regress/input/json_load.source
----------------------------------------------------------------------
diff --git a/src/test/regress/input/json_load.source b/src/test/regress/input/json_load.source
index 6dcef8a..c6f7120 100644
--- a/src/test/regress/input/json_load.source
+++ b/src/test/regress/input/json_load.source
@@ -29,7 +29,7 @@ CREATE OR REPLACE FUNCTION search_policy(tblname text)
AS '@abs_builddir@/regress@DLSUFFIX@', 'caql_scan_in_memory_gp_distribution_policy';
CREATE OR REPLACE FUNCTION search_exttable(tblname text)
- RETURNS table(oid "oid", relname text, location text)
+ RETURNS table(oid "oid", relname text, location text, fmttype text, fmtopts text)
LANGUAGE C volatile NO SQL
AS '@abs_builddir@/regress@DLSUFFIX@', 'caql_scan_in_memory_pg_exttable';
@@ -59,11 +59,17 @@ SELECT nspname, nspdboid, count(distinct oid) from search_namespace('default') g
SELECT relname, relkind from search_table('mytable');
SELECT relname from search_type('mytable');
SELECT relname, policy_atts from search_policy('mytable');
-SELECT relname, location from search_exttable('mytable');
+SELECT relname, location, fmttype, fmtopts from search_exttable('mytable');
SELECT relname, relkind from search_table('mytable') where oid >= min_external_oid();
SELECT * from search_attribute('mytable');
END TRANSACTION;
+-- positive test case with a single table, using text format and custom delimiter
+BEGIN TRANSACTION;
+SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table_text.json');
+SELECT relname, location, fmttype, fmtopts from search_exttable('mytable');
+END TRANSACTION;
+
-- positive test case with multiple tables
BEGIN TRANSACTION;
SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json');
@@ -73,7 +79,7 @@ SELECT nspname, nspdboid, count(distinct oid) from search_namespace('db2') where
SELECT relname, relkind from search_table('ht1') where oid >= min_external_oid();
SELECT relname, count(oid) from search_type('ht1') group by relname, namespace;
SELECT relname, policy_atts from search_policy('ht1');
-SELECT relname, location from search_exttable('ht1');
+SELECT relname, location, fmttype, fmtopts from search_exttable('ht1');
END TRANSACTION;
-- negative test: duplicated tables
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/src/test/regress/json_utils.c
----------------------------------------------------------------------
diff --git a/src/test/regress/json_utils.c b/src/test/regress/json_utils.c
index 77dc0b5..0028447 100644
--- a/src/test/regress/json_utils.c
+++ b/src/test/regress/json_utils.c
@@ -26,6 +26,7 @@
* number of output columns for the UDFs for scanning in memory catalog tables
*/
#define NUM_COLS 3
+#define NUM_COLS_EXTTABLE 5
static
char *read_file(const char *filename);
@@ -430,13 +431,17 @@ caql_scan_in_memory_pg_exttable(PG_FUNCTION_ARGS)
/* switch context when allocating stuff to be used in later calls */
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
- tupdesc = CreateTemplateTupleDesc(NUM_COLS, false);
+ tupdesc = CreateTemplateTupleDesc(NUM_COLS_EXTTABLE, false);
TupleDescInitEntry(tupdesc, (AttrNumber) 1, "exttableoid",
OIDOID, -1, 0);
TupleDescInitEntry(tupdesc, (AttrNumber) 2, "exttablename",
TEXTOID, -1, 0);
TupleDescInitEntry(tupdesc, (AttrNumber) 3, "exttablelocation",
TEXTOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 4, "exttableformat",
+ TEXTOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 5, "exttableformatoptions",
+ TEXTOID, -1, 0);
funcctx->tuple_desc = BlessTupleDesc(tupdesc);
/* iterate on pg_class and query
@@ -458,8 +463,8 @@ caql_scan_in_memory_pg_exttable(PG_FUNCTION_ARGS)
if (NULL != (pgclasstup = caql_getnext(pcqCtx)))
{
- Datum values[NUM_COLS];
- bool nulls[NUM_COLS];
+ Datum values[NUM_COLS_EXTTABLE];
+ bool nulls[NUM_COLS_EXTTABLE];
/* create tuples for pg_exttable table */
cqContext* pcqCtx1 = caql_beginscan(
@@ -482,6 +487,8 @@ caql_scan_in_memory_pg_exttable(PG_FUNCTION_ARGS)
nulls[1] = false;
Datum locations = tuple_getattr(readtup, tupleDesc, Anum_pg_exttable_location);
+ Datum fmttype = tuple_getattr(readtup, tupleDesc, Anum_pg_exttable_fmttype);
+ Datum fmtopts = tuple_getattr(readtup, tupleDesc, Anum_pg_exttable_fmtopts);
Datum* elems = NULL;
int nelems;
deconstruct_array(DatumGetArrayTypeP(locations),
@@ -490,10 +497,19 @@ caql_scan_in_memory_pg_exttable(PG_FUNCTION_ARGS)
Assert(nelems > 0);
char *loc_str = DatumGetCString(DirectFunctionCall1(textout, elems[0]));
text *t2 = cstring_to_text(loc_str);
-
values[2] = PointerGetDatum(t2);
nulls[2] = false;
+ char fmttype_chr = DatumGetChar(fmttype);
+ text *t3 = cstring_to_text_with_len(&fmttype_chr, 1);
+ values[3] = PointerGetDatum(t3);
+ nulls[3] = false;
+
+ char *fmtopts_str = DatumGetCString(fmtopts);
+ text *t4 = cstring_to_text(fmtopts_str);
+ values[4] = PointerGetDatum(t4);
+ nulls[4] = false;
+
/* build tuple */
restuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6fa1ced2/src/test/regress/output/json_load.source
----------------------------------------------------------------------
diff --git a/src/test/regress/output/json_load.source b/src/test/regress/output/json_load.source
index 5bec43e..1717603 100644
--- a/src/test/regress/output/json_load.source
+++ b/src/test/regress/output/json_load.source
@@ -22,7 +22,7 @@ CREATE OR REPLACE FUNCTION search_policy(tblname text)
LANGUAGE C volatile NO SQL
AS '@abs_builddir@/regress@DLSUFFIX@', 'caql_scan_in_memory_gp_distribution_policy';
CREATE OR REPLACE FUNCTION search_exttable(tblname text)
- RETURNS table(oid "oid", relname text, location text)
+ RETURNS table(oid "oid", relname text, location text, fmttype text, fmtopts text)
LANGUAGE C volatile NO SQL
AS '@abs_builddir@/regress@DLSUFFIX@', 'caql_scan_in_memory_pg_exttable';
CREATE OR REPLACE FUNCTION search_attribute(tblname text)
@@ -99,10 +99,10 @@ SELECT relname, policy_atts from search_policy('mytable');
mytable | null
(1 row)
-SELECT relname, location from search_exttable('mytable');
- relname | location
----------+----------------------------------------------------
- mytable | pxf://localhost:51200/default.mytable?Profile=Hive
+SELECT relname, location, fmttype, fmtopts from search_exttable('mytable');
+ relname | location | fmttype | fmtopts
+---------+-------------------------------------------------------------------+---------+--------------------------------
+ mytable | pxf://localhost:51200/default.mytable?Profile=Hive&delimiter=\x01 | b | formatter 'pxfwritable_import'
(1 row)
SELECT relname, relkind from search_table('mytable') where oid >= min_external_oid();
@@ -132,6 +132,21 @@ SELECT * from search_attribute('mytable');
(15 rows)
END TRANSACTION;
+-- positive test case with a single table, using text format and custom delimiter
+BEGIN TRANSACTION;
+SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table_text.json');
+ load_json_data
+------------------
+ default.mytable
+(1 row)
+
+SELECT relname, location, fmttype, fmtopts from search_exttable('mytable');
+ relname | location | fmttype | fmtopts
+---------+-------------------------------------------------------------------+---------+------------------------------------
+ mytable | pxf://localhost:51200/default.mytable?Profile=Hive&delimiter=\x2a | t | delimiter '*' null '\N' escape '\'
+(1 row)
+
+END TRANSACTION;
-- positive test case with multiple tables
BEGIN TRANSACTION;
SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json');
@@ -173,11 +188,11 @@ SELECT relname, policy_atts from search_policy('ht1');
ht1 | null
(2 rows)
-SELECT relname, location from search_exttable('ht1');
- relname | location
----------+--------------------------------------------
- ht1 | pxf://localhost:51200/db1.ht1?Profile=Hive
- ht1 | pxf://localhost:51200/db2.ht1?Profile=Hive
+SELECT relname, location, fmttype, fmtopts from search_exttable('ht1');
+ relname | location | fmttype | fmtopts
+---------+--------------------------------------------+---------+--------------------------------
+ ht1 | pxf://localhost:51200/db1.ht1?Profile=Hive | b | formatter 'pxfwritable_import'
+ ht1 | pxf://localhost:51200/db2.ht1?Profile=Hive | b | formatter 'pxfwritable_import'
(2 rows)
END TRANSACTION;