You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by sh...@apache.org on 2018/02/07 00:26:11 UTC
incubator-hawq git commit: HAWQ-1581. Separate PXF system parameters
from user parameters
Repository: incubator-hawq
Updated Branches:
refs/heads/master ba7923d8d -> 23845381e
HAWQ-1581. Separate PXF system parameters from user parameters
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/23845381
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/23845381
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/23845381
Branch: refs/heads/master
Commit: 23845381e940bdb193155afa3c8af47b3a7100c4
Parents: ba7923d
Author: shivzone <sh...@gmail.com>
Authored: Tue Feb 6 16:26:05 2018 -0800
Committer: shivzone <sh...@gmail.com>
Committed: Tue Feb 6 16:26:05 2018 -0800
----------------------------------------------------------------------
.../hawq/pxf/api/utilities/InputData.java | 9 ++-
.../hawq/pxf/api/utilities/ProfilesConf.java | 2 +-
.../pxf/api/utilities/ProfilesConfTest.java | 12 ++--
.../pxf/plugins/hive/HiveDataFragmenter.java | 2 +-
.../pxf/plugins/hive/HiveMetadataFetcher.java | 2 +-
.../apache/hawq/pxf/plugins/json/PxfUnit.java | 16 ++---
.../pxf/service/utilities/ProtocolData.java | 32 ++++++----
pxf/pxf-service/src/scripts/pxf-service | 19 +++---
.../pxf/service/BridgeOutputBuilderTest.java | 14 ++---
.../pxf/service/utilities/ProtocolDataTest.java | 66 ++++++++++----------
src/backend/access/external/pxfuriparser.c | 6 +-
.../access/external/test/pxfuriparser_test.c | 4 +-
src/bin/gpfusion/gpbridgeapi.c | 4 +-
13 files changed, 99 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/23845381/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
index 7bf57ed..de9d8e9 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
@@ -23,15 +23,18 @@ package org.apache.hawq.pxf.api.utilities;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import java.util.*;
+import java.util.Map;
+import java.util.ArrayList;
+
/**
* Common configuration available to all PXF plugins. Represents input data
- * coming from client applications, such as Hawq.
+ * coming from client applications, such as Hawq or GPDB.
*/
public class InputData {
public static final String DELIMITER_KEY = "DELIMITER";
+ public static final String USER_PROP_PREFIX = "X-GP-OPTIONS-";
public static final int INVALID_SPLIT_IDX = -1;
private static final Log LOG = LogFactory.getLog(InputData.class);
@@ -125,7 +128,7 @@ public class InputData {
* @return property value as a String
*/
public String getUserProperty(String userProp) {
- return requestParametersMap.get("X-GP-" + userProp.toUpperCase());
+ return requestParametersMap.get(USER_PROP_PREFIX + userProp.toUpperCase());
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/23845381/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/ProfilesConf.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/ProfilesConf.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/ProfilesConf.java
index 2c20ab7..e3e8b71 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/ProfilesConf.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/ProfilesConf.java
@@ -129,7 +129,7 @@ public enum ProfilesConf {
for (String plugin : plugins) {
String pluginValue = profileSubset.getString(plugin);
if (!StringUtils.isEmpty(StringUtils.trim(pluginValue))) {
- pluginsMap.put("X-GP-" + plugin.toUpperCase(), pluginValue);
+ pluginsMap.put(InputData.USER_PROP_PREFIX + plugin.toUpperCase(), pluginValue);
}
}
return pluginsMap;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/23845381/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/ProfilesConfTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/ProfilesConfTest.java b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/ProfilesConfTest.java
index 0631cb2..32093cc 100644
--- a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/ProfilesConfTest.java
+++ b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/utilities/ProfilesConfTest.java
@@ -97,15 +97,15 @@ class ProfilesConfTestDefinedProfile extends ProfilesConfTest {
Map<String, String> hbaseProfile = ProfilesConf.getProfilePluginsMap("HBase");
assertEquals(2, hbaseProfile.keySet().size());
- assertEquals(hbaseProfile.get("X-GP-PLUGIN1"), "X");
- assertEquals(hbaseProfile.get("X-GP-PLUGIN2"), "XX");
+ assertEquals(hbaseProfile.get("X-GP-OPTIONS-PLUGIN1"), "X");
+ assertEquals(hbaseProfile.get("X-GP-OPTIONS-PLUGIN2"), "XX");
Map<String, String> hiveProfile = ProfilesConf.getProfilePluginsMap("hIVe");// case
// insensitive
// profile
// name
assertEquals(1, hiveProfile.keySet().size());
- assertEquals(hiveProfile.get("X-GP-PLUGIN1"), "Y");
+ assertEquals(hiveProfile.get("X-GP-OPTIONS-PLUGIN1"), "Y");
Mockito.verify(LOG).info("PXF profiles loaded: [HBase, Hive]");
}
@@ -170,8 +170,8 @@ class ProfilesConfTestOverrideProfile extends ProfilesConfTest {
optionalFile.toURI().toURL());
Map profile = ProfilesConf.getProfilePluginsMap("HBase");
assertEquals(2, profile.keySet().size());
- assertEquals(profile.get("X-GP-PLUGIN1"), "Y");
- assertEquals(profile.get("X-GP-PLUGIN2"), "YY");
+ assertEquals(profile.get("X-GP-OPTIONS-PLUGIN1"), "Y");
+ assertEquals(profile.get("X-GP-OPTIONS-PLUGIN2"), "YY");
}
}
@@ -242,7 +242,7 @@ class ProfilesConfTestMissingOptionalProfileFile extends ProfilesConfTest {
mandatoryFile.toURI().toURL());
when(classLoader.getResource("pxf-profiles.xml")).thenReturn(null);
Map<String, String> hbaseProfile = ProfilesConf.getProfilePluginsMap("HBase");
- assertEquals("Y", hbaseProfile.get("X-GP-PLUGIN1"));
+ assertEquals("Y", hbaseProfile.get("X-GP-OPTIONS-PLUGIN1"));
Mockito.verify(LOG).warn("pxf-profiles.xml not found in the classpath");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/23845381/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
index 76b83e6..406630c 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
@@ -299,7 +299,7 @@ public class HiveDataFragmenter extends Fragmenter {
}
String fragmenterForProfile = null;
if (profile != null) {
- fragmenterForProfile = ProfilesConf.getProfilePluginsMap(profile).get("X-GP-FRAGMENTER");
+ fragmenterForProfile = ProfilesConf.getProfilePluginsMap(profile).get("X-GP-OPTIONS-FRAGMENTER");
} else {
fragmenterForProfile = inputData.getFragmenter();
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/23845381/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
index dc76289..2a27b88 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
@@ -137,7 +137,7 @@ public class HiveMetadataFetcher extends MetadataFetcher {
OutputFormat outputFormat = null;
InputFormat<?, ?> fformat = HiveDataFragmenter.makeInputFormat(inputFormat, jobConf);
String profile = ProfileFactory.get(fformat, hasComplexTypes);
- String outputFormatClassName = ProfilesConf.getProfilePluginsMap(profile).get("X-GP-OUTPUTFORMAT");
+ String outputFormatClassName = ProfilesConf.getProfilePluginsMap(profile).get("X-GP-OPTIONS-OUTPUTFORMAT");
outputFormat = OutputFormat.getOutputFormat(outputFormatClassName);
return outputFormat;
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/23845381/pxf/pxf-json/src/test/java/org/apache/hawq/pxf/plugins/json/PxfUnit.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-json/src/test/java/org/apache/hawq/pxf/plugins/json/PxfUnit.java b/pxf/pxf-json/src/test/java/org/apache/hawq/pxf/plugins/json/PxfUnit.java
index 5669882..c9ba4d0 100644
--- a/pxf/pxf-json/src/test/java/org/apache/hawq/pxf/plugins/json/PxfUnit.java
+++ b/pxf/pxf-json/src/test/java/org/apache/hawq/pxf/plugins/json/PxfUnit.java
@@ -246,7 +246,7 @@ public abstract class PxfUnit {
}
/**
- * Get any extra parameters that are meant to be specified for the "pxf" protocol. Note that "X-GP-" is prepended to
+ * Get any extra parameters that are meant to be specified for the "pxf" protocol. Note that "X-GP-OPTIONS-" is prepended to
* each parameter name.
*
* @return Any extra parameters or null if none.
@@ -301,12 +301,12 @@ public abstract class PxfUnit {
paramsMap.put("X-GP-ATTR-TYPECODE" + i, Integer.toString(params.get(i).second.getOID()));
}
- paramsMap.put("X-GP-ACCESSOR", getWriteAccessorClass().getName());
- paramsMap.put("X-GP-RESOLVER", getWriteResolverClass().getName());
+ paramsMap.put("X-GP-OPTIONS-ACCESSOR", getWriteAccessorClass().getName());
+ paramsMap.put("X-GP-OPTIONS-RESOLVER", getWriteResolverClass().getName());
if (getExtraParams() != null) {
for (Pair<String, String> param : getExtraParams()) {
- paramsMap.put("X-GP-" + param.first, param.second);
+ paramsMap.put("X-GP-OPTIONS-" + param.first, param.second);
}
}
@@ -342,7 +342,6 @@ public abstract class PxfUnit {
paramsMap.put("X-GP-SEGMENT-ID", "1");
paramsMap.put("X-GP-HAS-FILTER", "0");
paramsMap.put("X-GP-SEGMENT-COUNT", "1");
- paramsMap.put("X-GP-FRAGMENTER", getFragmenterClass().getName());
paramsMap.put("X-GP-FORMAT", "GPDBWritable");
paramsMap.put("X-GP-URL-HOST", "localhost");
paramsMap.put("X-GP-URL-PORT", "50070");
@@ -358,12 +357,13 @@ public abstract class PxfUnit {
}
// HDFSMetaData properties
- paramsMap.put("X-GP-ACCESSOR", getReadAccessorClass().getName());
- paramsMap.put("X-GP-RESOLVER", getReadResolverClass().getName());
+ paramsMap.put("X-GP-OPTIONS-FRAGMENTER", getFragmenterClass().getName());
+ paramsMap.put("X-GP-OPTIONS-ACCESSOR", getReadAccessorClass().getName());
+ paramsMap.put("X-GP-OPTIONS-RESOLVER", getReadResolverClass().getName());
if (getExtraParams() != null) {
for (Pair<String, String> param : getExtraParams()) {
- paramsMap.put("X-GP-" + param.first, param.second);
+ paramsMap.put("X-GP-OPTIONS-" + param.first, param.second);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/23845381/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
index 65cdf16..1688982 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
@@ -57,8 +57,9 @@ public class ProtocolData extends InputData {
protected float statsSampleRatio;
/**
- * Constructs a ProtocolData. Parses X-GP-* configuration variables.
- *
+ * Constructs a ProtocolData.
+ * Parses X-GP-* system configuration variables and
+ * X-GP-OPTIONS-* user configuration variables
* @param paramsMap contains all query-specific parameters from Hawq
*/
public ProtocolData(Map<String, String> paramsMap) {
@@ -82,19 +83,26 @@ public class ProtocolData extends InputData {
parseTupleDescription();
/*
- * accessor - will throw exception from getPropery() if outputFormat is
+ * accessor - will throw exception if outputFormat is
* BINARY and the user did not supply accessor=... or profile=...
- * resolver - will throw exception from getPropery() if outputFormat is
+ * resolver - will throw exception if outputFormat is
* BINARY and the user did not supply resolver=... or profile=...
*/
- profile = getOptionalProperty("PROFILE");
+ profile = getUserProperty("PROFILE");
if (profile != null) {
setProfilePlugins();
}
- accessor = getProperty("ACCESSOR");
- resolver = getProperty("RESOLVER");
- fragmenter = getOptionalProperty("FRAGMENTER");
- metadata = getOptionalProperty("METADATA");
+ accessor = getUserProperty("ACCESSOR");
+ if(accessor == null) {
+ protocolViolation("ACCESSOR");
+ }
+ resolver = getUserProperty("RESOLVER");
+ if(resolver == null) {
+ protocolViolation("RESOLVER");
+ }
+
+ fragmenter = getUserProperty("FRAGMENTER");
+ metadata = getUserProperty("METADATA");
dataSource = getProperty("DATA-DIR");
/* Kerberos token information */
@@ -207,7 +215,7 @@ public class ProtocolData extends InputData {
if (!duplicates.isEmpty()) {
throw new IllegalArgumentException("Profile '" + profile
+ "' already defines: "
- + String.valueOf(duplicates).replace("X-GP-", ""));
+ + String.valueOf(duplicates).replace("X-GP-OPTIONS-", ""));
}
}
@@ -354,7 +362,7 @@ public class ProtocolData extends InputData {
private void parseThreadSafe() {
threadSafe = true;
- String threadSafeStr = getOptionalProperty("THREAD-SAFE");
+ String threadSafeStr = getUserProperty("THREAD-SAFE");
if (threadSafeStr != null) {
threadSafe = parseBooleanValue(threadSafeStr);
}
@@ -491,7 +499,7 @@ public class ProtocolData extends InputData {
private void parseStatsParameters() {
- String maxFrags = getOptionalProperty("STATS-MAX-FRAGMENTS");
+ String maxFrags = getUserProperty("STATS-MAX-FRAGMENTS");
if (!StringUtils.isEmpty(maxFrags)) {
statsMaxFragments = Integer.parseInt(maxFrags);
if (statsMaxFragments <= 0) {
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/23845381/pxf/pxf-service/src/scripts/pxf-service
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/scripts/pxf-service b/pxf/pxf-service/src/scripts/pxf-service
index d061279..13fcf44 100644
--- a/pxf/pxf-service/src/scripts/pxf-service
+++ b/pxf/pxf-service/src/scripts/pxf-service
@@ -20,7 +20,6 @@
# pxf-service start/stop/initialize/status the PXF instance
#
-
if [ -z ${PXF_HOME} ]; then
parent_script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
env_script=${parent_script_dir}/conf/pxf-env.sh
@@ -197,26 +196,26 @@ function configureWebapp()
popd > /dev/null
context_file=${instance_root}/${instance_name}/webapps/pxf/META-INF/context.xml
- sed -i -e "s:classpathFiles=\"[a-zA-Z0-9\/\;.-]*\":classpathFiles=\"${PXF_HOME}\/conf\/pxf-private.classpath\":" \
- -e "s:secondaryClasspathFiles=\"[a-zA-Z0-9\/\;.-]*\":secondaryClasspathFiles=\"${PXF_HOME}\/conf\/pxf-public.classpath\":" ${context_file}
+ sed -i -e "s:classpathFiles=\"[a-zA-Z0-9\/\;.-]*\":classpathFiles=\"${PXF_HOME}\/conf\/pxf-private.classpath\":" ${context_file}
+ sed -i -e "s:secondaryClasspathFiles=\"[a-zA-Z0-9\/\;.-]*\":secondaryClasspathFiles=\"${PXF_HOME}\/conf\/pxf-public.classpath\":" ${context_file}
web_file=${instance_root}/${instance_name}/webapps/pxf/WEB-INF/web.xml
- sed -i "s:<param-value>.*pxf-log4j.properties<\/param-value>:<param-value>$PXF_HOME\/conf\/pxf-log4j.properties<\/param-value>:" ${web_file}
+ sed -i -e "s:<param-value>.*pxf-log4j.properties<\/param-value>:<param-value>$PXF_HOME\/conf\/pxf-log4j.properties<\/param-value>:" ${web_file}
# set port
catalinaProperties=${instance_root}/${instance_name}/conf/catalina.properties
- sed -i "s|^[[:blank:]]*connector.http.port=.*$|connector.http.port=$instance_port|g" ${catalinaProperties}
+ sed -i -e "s|^[[:blank:]]*connector.http.port=.*$|connector.http.port=$instance_port|g" ${catalinaProperties}
# set container configs
catalinaEnv=${instance_root}/${instance_name}/bin/setenv.sh
- sed -i -e "s|JVM_OPTS=.*$|JVM_OPTS=\"${PXF_JVM_OPTS}\"|g" \
- -e "s|-Dpxf.log.dir=[^[:space:]^\"]*|-Dpxf.log.dir=${PXF_LOGDIR} |g" \
- -e "s|^[[:blank:]]*CATALINA_PID=.*$|CATALINA_PID=${PXF_RUNDIR}/catalina.pid|g" \
- -e "s|^[[:blank:]]*CATALINA_OUT=.*$|CATALINA_OUT=${PXF_LOGDIR}/catalina.out|g" ${catalinaEnv}
+ sed -i -e "s|JVM_OPTS=.*$|JVM_OPTS=\"${PXF_JVM_OPTS}\"|g" ${catalinaEnv}
+ sed -i -e "s|-Dpxf.log.dir=[^[:space:]^\"]*|-Dpxf.log.dir=${PXF_LOGDIR} |g" ${catalinaEnv}
+ sed -i -e "s|^[[:blank:]]*CATALINA_PID=.*$|CATALINA_PID=${PXF_RUNDIR}/catalina.pid|g" ${catalinaEnv}
+ sed -i -e "s|^[[:blank:]]*CATALINA_OUT=.*$|CATALINA_OUT=${PXF_LOGDIR}/catalina.out|g" ${catalinaEnv}
# set log directories
catalinaLog=${instance_root}/$instance_name/conf/logging.properties
- sed -i "s|juli.FileHandler.directory\s*=.*$|juli.FileHandler.directory = ${PXF_LOGDIR}|g" ${catalinaLog}
+ sed -i -e "s|juli.FileHandler.directory\s*=.*$|juli.FileHandler.directory = ${PXF_LOGDIR}|g" ${catalinaLog}
}
function commandWebapp()
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/23845381/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/BridgeOutputBuilderTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/BridgeOutputBuilderTest.java b/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/BridgeOutputBuilderTest.java
index a00910d..9dbd906 100644
--- a/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/BridgeOutputBuilderTest.java
+++ b/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/BridgeOutputBuilderTest.java
@@ -324,8 +324,8 @@ public class BridgeOutputBuilderTest {
parameters.put("X-GP-ATTRS", "1");
addColumn(parameters, 0, DataType.TEXT, "col0");
// activate sampling code
- parameters.put("X-GP-STATS-MAX-FRAGMENTS", "100");
- parameters.put("X-GP-STATS-SAMPLE-RATIO", "1.00");
+ parameters.put("X-GP-OPTIONS-STATS-MAX-FRAGMENTS", "100");
+ parameters.put("X-GP-OPTIONS-STATS-SAMPLE-RATIO", "1.00");
BridgeOutputBuilder builder = makeBuilder(parameters);
LinkedList<Writable> outputQueue = builder.makeOutput(fields);
@@ -352,8 +352,8 @@ public class BridgeOutputBuilderTest {
parameters.put("X-GP-ATTRS", "1");
addColumn(parameters, 0, DataType.TEXT, "col0");
// activate sampling code
- parameters.put("X-GP-STATS-MAX-FRAGMENTS", "100");
- parameters.put("X-GP-STATS-SAMPLE-RATIO", "1.00");
+ parameters.put("X-GP-OPTIONS-STATS-MAX-FRAGMENTS", "100");
+ parameters.put("X-GP-OPTIONS-STATS-SAMPLE-RATIO", "1.00");
BridgeOutputBuilder builder = makeBuilder(parameters);
LinkedList<Writable> outputQueue = builder.makeOutput(fields);
@@ -464,11 +464,11 @@ public class BridgeOutputBuilderTest {
parameters.put("X-GP-FORMAT", "TEXT");
parameters.put("X-GP-URL-HOST", "my://bags");
parameters.put("X-GP-URL-PORT", "-8020");
- parameters.put("X-GP-ACCESSOR", "are");
- parameters.put("X-GP-RESOLVER", "packed");
+ parameters.put("X-GP-OPTIONS-ACCESSOR", "are");
+ parameters.put("X-GP-OPTIONS-RESOLVER", "packed");
parameters.put("X-GP-DATA-DIR", "i'm/ready/to/go");
parameters.put("X-GP-FRAGMENT-METADATA", "U29tZXRoaW5nIGluIHRoZSB3YXk=");
- parameters.put("X-GP-I'M-STANDING-HERE", "outside-your-door");
+ parameters.put("X-GP-OPTIONS-I'M-STANDING-HERE", "outside-your-door");
ProtocolData protocolData = new ProtocolData(parameters);
BridgeOutputBuilder builder = new BridgeOutputBuilder(protocolData);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/23845381/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/utilities/ProtocolDataTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/utilities/ProtocolDataTest.java b/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/utilities/ProtocolDataTest.java
index 10479c7..6cbd2f1 100644
--- a/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/utilities/ProtocolDataTest.java
+++ b/pxf/pxf-service/src/test/java/org/apache/hawq/pxf/service/utilities/ProtocolDataTest.java
@@ -94,7 +94,7 @@ public class ProtocolDataTest {
when(ProfilesConf.getProfilePluginsMap("a profile")).thenReturn(
mockedProfiles);
- parameters.put("x-gp-profile", "a profile");
+ parameters.put("x-gp-options-profile", "a profile");
parameters.put("when you try your best", "and you do succeed");
parameters.put("WHEN you GET what YOU want", "and what you need");
@@ -110,9 +110,9 @@ public class ProtocolDataTest {
@Test
public void definedProfile() throws Exception {
- parameters.put("X-GP-PROFILE", "HIVE");
- parameters.remove("X-GP-ACCESSOR");
- parameters.remove("X-GP-RESOLVER");
+ parameters.put("X-GP-OPTIONS-PROFILE", "HIVE");
+ parameters.remove("X-GP-OPTIONS-ACCESSOR");
+ parameters.remove("X-GP-OPTIONS-RESOLVER");
ProtocolData protocolData = new ProtocolData(parameters);
assertEquals(protocolData.getFragmenter(), "org.apache.hawq.pxf.plugins.hive.HiveDataFragmenter");
assertEquals(protocolData.getAccessor(), "org.apache.hawq.pxf.plugins.hive.HiveAccessor");
@@ -121,7 +121,7 @@ public class ProtocolDataTest {
@Test
public void undefinedProfile() throws Exception {
- parameters.put("X-GP-PROFILE", "THIS_PROFILE_NEVER_EXISTED!");
+ parameters.put("X-GP-OPTIONS-PROFILE", "THIS_PROFILE_NEVER_EXISTED!");
try {
new ProtocolData(parameters);
fail("Undefined profile should throw ProfileConfException");
@@ -132,29 +132,29 @@ public class ProtocolDataTest {
@Test
public void threadSafeTrue() throws Exception {
- parameters.put("X-GP-THREAD-SAFE", "TRUE");
+ parameters.put("X-GP-OPTIONS-THREAD-SAFE", "TRUE");
ProtocolData protocolData = new ProtocolData(parameters);
assertEquals(protocolData.isThreadSafe(), true);
- parameters.put("X-GP-THREAD-SAFE", "true");
+ parameters.put("X-GP-OPTIONS-THREAD-SAFE", "true");
protocolData = new ProtocolData(parameters);
assertEquals(protocolData.isThreadSafe(), true);
}
@Test
public void threadSafeFalse() throws Exception {
- parameters.put("X-GP-THREAD-SAFE", "False");
+ parameters.put("X-GP-OPTIONS-THREAD-SAFE", "False");
ProtocolData protocolData = new ProtocolData(parameters);
assertEquals(protocolData.isThreadSafe(), false);
- parameters.put("X-GP-THREAD-SAFE", "falSE");
+ parameters.put("X-GP-OPTIONS-THREAD-SAFE", "falSE");
protocolData = new ProtocolData(parameters);
assertEquals(protocolData.isThreadSafe(), false);
}
@Test
public void threadSafeMaybe() throws Exception {
- parameters.put("X-GP-THREAD-SAFE", "maybe");
+ parameters.put("X-GP-OPTIONS-THREAD-SAFE", "maybe");
try {
new ProtocolData(parameters);
fail("illegal THREAD-SAFE value should throw IllegalArgumentException");
@@ -166,7 +166,7 @@ public class ProtocolDataTest {
@Test
public void threadSafeDefault() throws Exception {
- parameters.remove("X-GP-THREAD-SAFE");
+ parameters.remove("X-GP-OPTIONS-THREAD-SAFE");
ProtocolData protocolData = new ProtocolData(parameters);
assertEquals(protocolData.isThreadSafe(), true);
}
@@ -232,8 +232,8 @@ public class ProtocolDataTest {
@Test
public void statsParams() {
- parameters.put("X-GP-STATS-MAX-FRAGMENTS", "10101");
- parameters.put("X-GP-STATS-SAMPLE-RATIO", "0.039");
+ parameters.put("X-GP-OPTIONS-STATS-MAX-FRAGMENTS", "10101");
+ parameters.put("X-GP-OPTIONS-STATS-SAMPLE-RATIO", "0.039");
ProtocolData protData = new ProtocolData(parameters);
@@ -243,21 +243,21 @@ public class ProtocolDataTest {
@Test
public void statsMissingParams() {
- parameters.put("X-GP-STATS-MAX-FRAGMENTS", "13");
+ parameters.put("X-GP-OPTIONS-STATS-MAX-FRAGMENTS", "13");
try {
new ProtocolData(parameters);
- fail("missing X-GP-STATS-SAMPLE-RATIO parameter");
+ fail("missing X-GP-OPTIONS-STATS-SAMPLE-RATIO parameter");
} catch (IllegalArgumentException e) {
assertEquals(
e.getMessage(),
"Missing parameter: STATS-SAMPLE-RATIO and STATS-MAX-FRAGMENTS must be set together");
}
- parameters.remove("X-GP-STATS-MAX-FRAGMENTS");
- parameters.put("X-GP-STATS-SAMPLE-RATIO", "1");
+ parameters.remove("X-GP-OPTIONS-STATS-MAX-FRAGMENTS");
+ parameters.put("X-GP-OPTIONS-STATS-SAMPLE-RATIO", "1");
try {
new ProtocolData(parameters);
- fail("missing X-GP-STATS-MAX-FRAGMENTS parameter");
+ fail("missing X-GP-OPTIONS-STATS-MAX-FRAGMENTS parameter");
} catch (IllegalArgumentException e) {
assertEquals(
e.getMessage(),
@@ -267,11 +267,11 @@ public class ProtocolDataTest {
@Test
public void statsSampleRatioNegative() {
- parameters.put("X-GP-STATS-SAMPLE-RATIO", "101");
+ parameters.put("X-GP-OPTIONS-STATS-SAMPLE-RATIO", "101");
try {
new ProtocolData(parameters);
- fail("wrong X-GP-STATS-SAMPLE-RATIO value");
+ fail("wrong X-GP-OPTIONS-STATS-SAMPLE-RATIO value");
} catch (IllegalArgumentException e) {
assertEquals(
e.getMessage(),
@@ -279,10 +279,10 @@ public class ProtocolDataTest {
+ "STATS-SAMPLE-RATIO must be a value between 0.0001 and 1.0");
}
- parameters.put("X-GP-STATS-SAMPLE-RATIO", "0");
+ parameters.put("X-GP-OPTIONS-STATS-SAMPLE-RATIO", "0");
try {
new ProtocolData(parameters);
- fail("wrong X-GP-STATS-SAMPLE-RATIO value");
+ fail("wrong X-GP-OPTIONS-STATS-SAMPLE-RATIO value");
} catch (IllegalArgumentException e) {
assertEquals(
e.getMessage(),
@@ -290,10 +290,10 @@ public class ProtocolDataTest {
+ "STATS-SAMPLE-RATIO must be a value between 0.0001 and 1.0");
}
- parameters.put("X-GP-STATS-SAMPLE-RATIO", "0.00005");
+ parameters.put("X-GP-OPTIONS-STATS-SAMPLE-RATIO", "0.00005");
try {
new ProtocolData(parameters);
- fail("wrong X-GP-STATS-SAMPLE-RATIO value");
+ fail("wrong X-GP-OPTIONS-STATS-SAMPLE-RATIO value");
} catch (IllegalArgumentException e) {
assertEquals(
e.getMessage(),
@@ -301,10 +301,10 @@ public class ProtocolDataTest {
+ "STATS-SAMPLE-RATIO must be a value between 0.0001 and 1.0");
}
- parameters.put("X-GP-STATS-SAMPLE-RATIO", "a");
+ parameters.put("X-GP-OPTIONS-STATS-SAMPLE-RATIO", "a");
try {
new ProtocolData(parameters);
- fail("wrong X-GP-STATS-SAMPLE-RATIO value");
+ fail("wrong X-GP-OPTIONS-STATS-SAMPLE-RATIO value");
} catch (NumberFormatException e) {
assertEquals(e.getMessage(), "For input string: \"a\"");
}
@@ -312,20 +312,20 @@ public class ProtocolDataTest {
@Test
public void statsMaxFragmentsNegative() {
- parameters.put("X-GP-STATS-MAX-FRAGMENTS", "10.101");
+ parameters.put("X-GP-OPTIONS-STATS-MAX-FRAGMENTS", "10.101");
try {
new ProtocolData(parameters);
- fail("wrong X-GP-STATS-MAX-FRAGMENTS value");
+ fail("wrong X-GP-OPTIONS-STATS-MAX-FRAGMENTS value");
} catch (NumberFormatException e) {
assertEquals(e.getMessage(), "For input string: \"10.101\"");
}
- parameters.put("X-GP-STATS-MAX-FRAGMENTS", "0");
+ parameters.put("X-GP-OPTIONS-STATS-MAX-FRAGMENTS", "0");
try {
new ProtocolData(parameters);
- fail("wrong X-GP-STATS-MAX-FRAGMENTS value");
+ fail("wrong X-GP-OPTIONS-STATS-MAX-FRAGMENTS value");
} catch (IllegalArgumentException e) {
assertEquals(e.getMessage(), "Wrong value '0'. "
+ "STATS-MAX-FRAGMENTS must be a positive integer");
@@ -425,11 +425,11 @@ public class ProtocolDataTest {
parameters.put("X-GP-URL-HOST", "my://bags");
parameters.put("X-GP-URL-PORT", "-8020");
parameters.put("X-GP-ATTRS", "-1");
- parameters.put("X-GP-ACCESSOR", "are");
- parameters.put("X-GP-RESOLVER", "packed");
+ parameters.put("X-GP-OPTIONS-ACCESSOR", "are");
+ parameters.put("X-GP-OPTIONS-RESOLVER", "packed");
parameters.put("X-GP-DATA-DIR", "i'm/ready/to/go");
parameters.put("X-GP-FRAGMENT-METADATA", "U29tZXRoaW5nIGluIHRoZSB3YXk=");
- parameters.put("X-GP-I'M-STANDING-HERE", "outside-your-door");
+ parameters.put("X-GP-OPTIONS-I'M-STANDING-HERE", "outside-your-door");
PowerMockito.mockStatic(UserGroupInformation.class);
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/23845381/src/backend/access/external/pxfuriparser.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/pxfuriparser.c b/src/backend/access/external/pxfuriparser.c
index 17d5803..595280c 100644
--- a/src/backend/access/external/pxfuriparser.c
+++ b/src/backend/access/external/pxfuriparser.c
@@ -530,7 +530,7 @@ GPHDUri_parse_option(char* pair, GPHDUri *uri)
option_data->value = pnstrdup(sep + 1, value_len);
char *x_gp_key = normalize_key_name(option_data->key);
- if (strcmp(x_gp_key, "X-GP-PROFILE") == 0)
+ if (strcmp(x_gp_key, "X-GP-OPTIONS-PROFILE") == 0)
{
uri->profile = pstrdup(option_data->value);
}
@@ -899,7 +899,7 @@ bool RelationIsExternalPxfReadOnly(Relation rel, StringInfo location)
/*
* Full name of the HEADER KEY expected by the PXF service
- * Converts input string to upper case and prepends "X-GP-" string
+ * Converts input string to upper case and prepends "X-GP-OPTIONS" string
*
*/
char* normalize_key_name(const char* key)
@@ -914,7 +914,7 @@ char* normalize_key_name(const char* key)
StringInfoData formatter;
initStringInfo(&formatter);
char* upperCasedKey = str_toupper(pstrdup(key), strlen(key));
- appendStringInfo(&formatter, "X-GP-%s", upperCasedKey);
+ appendStringInfo(&formatter, "X-GP-OPTIONS-%s", upperCasedKey);
pfree(upperCasedKey);
return formatter.data;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/23845381/src/backend/access/external/test/pxfuriparser_test.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/test/pxfuriparser_test.c b/src/backend/access/external/test/pxfuriparser_test.c
index 1912d2b..2efdf07 100644
--- a/src/backend/access/external/test/pxfuriparser_test.c
+++ b/src/backend/access/external/test/pxfuriparser_test.c
@@ -883,7 +883,7 @@ test__normalize_key_name_Positive(void **state)
{
char *input_key = strdup("mIxEdCaSeVaLuE");
char *normalized_key = normalize_key_name(input_key);
- assert_string_equal(normalized_key, "X-GP-MIXEDCASEVALUE");
+ assert_string_equal(normalized_key, "X-GP-OPTIONS-MIXEDCASEVALUE");
pfree(input_key);
pfree(normalized_key);
@@ -894,7 +894,7 @@ test__normalize_key_name_PositiveUpperCase(void **state)
{
char *input_key = strdup("ALREADY_UPPER_CASE");
char *normalized_key = normalize_key_name(input_key);
- assert_string_equal(normalized_key, "X-GP-ALREADY_UPPER_CASE");
+ assert_string_equal(normalized_key, "X-GP-OPTIONS-ALREADY_UPPER_CASE");
pfree(input_key);
pfree(normalized_key);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/23845381/src/bin/gpfusion/gpbridgeapi.c
----------------------------------------------------------------------
diff --git a/src/bin/gpfusion/gpbridgeapi.c b/src/bin/gpfusion/gpbridgeapi.c
index a853f06..2b5f221 100644
--- a/src/bin/gpfusion/gpbridgeapi.c
+++ b/src/bin/gpfusion/gpbridgeapi.c
@@ -234,13 +234,13 @@ void set_current_fragment_headers(gphadoop_context* context)
if (frag_data->profile)
{
/* if current fragment has optimal profile set it*/
- churl_headers_override(context->churl_headers, "X-GP-PROFILE", frag_data->profile);
+ churl_headers_override(context->churl_headers, "X-GP-OPTIONS-PROFILE", frag_data->profile);
elog(DEBUG2, "pxf: set_current_fragment_headers: using profile: %s", frag_data->profile);
} else if (context->gphd_uri->profile)
{
/* if current fragment doesn't have any optimal profile, set to use profile from url */
- churl_headers_override(context->churl_headers, "X-GP-PROFILE", context->gphd_uri->profile);
+ churl_headers_override(context->churl_headers, "X-GP-OPTIONS-PROFILE", context->gphd_uri->profile);
elog(DEBUG2, "pxf: set_current_fragment_headers: using profile: %s", context->gphd_uri->profile);
}
/* if there is no profile passed in url, we expect to have accessor+fragmenter+resolver so no action needed by this point */