You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by od...@apache.org on 2016/12/22 09:56:19 UTC
[1/2] incubator-hawq git commit: Revert "HAWQ-1178. Enhance
Fragmenter api to return profile name."
Repository: incubator-hawq
Updated Branches:
refs/heads/master fbef55d49 -> 90a69266e
Revert "HAWQ-1178. Enhance Fragmenter api to return profile name."
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2f8392e6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2f8392e6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2f8392e6
Branch: refs/heads/master
Commit: 2f8392e627cde75b9ababd74de6c8be125a3c595
Parents: fbef55d
Author: Oleksandr Diachenko <od...@pivotal.io>
Authored: Thu Dec 22 01:55:28 2016 -0800
Committer: Oleksandr Diachenko <od...@pivotal.io>
Committed: Thu Dec 22 01:55:28 2016 -0800
----------------------------------------------------------------------
pxf/build.gradle | 6 -
.../java/org/apache/hawq/pxf/api/Fragment.java | 22 ----
.../hawq/pxf/api/utilities/InputData.java | 11 --
.../pxf/plugins/hive/HiveDataFragmenter.java | 101 ++++++++++++---
.../plugins/hive/HiveInputFormatFragmenter.java | 71 ++++++++++
.../pxf/plugins/hive/HiveTablePartition.java | 56 --------
.../plugins/hive/utilities/HiveUtilities.java | 129 -------------------
.../apache/hawq/pxf/service/ProfileFactory.java | 45 -------
.../pxf/service/utilities/ProtocolData.java | 1 +
9 files changed, 153 insertions(+), 289 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2f8392e6/pxf/build.gradle
----------------------------------------------------------------------
diff --git a/pxf/build.gradle b/pxf/build.gradle
index 385bf08..aed48b1 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -193,12 +193,7 @@ project('pxf-service') {
providedCompile "org.apache.hadoop:hadoop-hdfs:$hadoopVersion"
providedCompile "org.apache.hadoop:hadoop-auth:$hadoopVersion"
providedCompile "org.apache.hadoop:hadoop-annotations:$hadoopVersion"
- providedCompile "org.apache.hadoop:hadoop-mapreduce-client-core:$hadoopVersion"
providedCompile "org.apache.tomcat:tomcat-catalina:$tomcatVersion"
- providedCompile("org.apache.hive:hive-exec:$hiveVersion") {
- exclude module: 'calcite-core'
- exclude module: 'calcite-avatica'
- }
}
ospackage {
@@ -347,7 +342,6 @@ project('pxf-hdfs') {
project('pxf-hive') {
dependencies {
compile(project(':pxf-hdfs'))
- compile(project(':pxf-service'))
compile("org.apache.hive:hive-exec:$hiveVersion") {
exclude module: 'calcite-core'
exclude module: 'calcite-avatica'
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2f8392e6/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
index ff6f131..130a91d 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/Fragment.java
@@ -51,11 +51,6 @@ public class Fragment {
private byte[] userData;
/**
- * Profile name, recommended for reading given Fragment.
- */
- private String profile;
-
- /**
* Constructs a Fragment.
*
* @param sourceName the resource uri (File path+name, table name, etc.)
@@ -88,15 +83,6 @@ public class Fragment {
this.userData = userData;
}
- public Fragment(String sourceName,
- String[] hosts,
- byte[] metadata,
- byte[] userData,
- String profile) {
- this(sourceName, hosts, metadata, userData);
- this.profile = profile;
- }
-
public String getSourceName() {
return sourceName;
}
@@ -132,12 +118,4 @@ public class Fragment {
public void setUserData(byte[] userData) {
this.userData = userData;
}
-
- public String getProfile() {
- return profile;
- }
-
- public void setProfile(String profile) {
- this.profile = profile;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2f8392e6/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
index 5afedca..891dba8 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
@@ -43,7 +43,6 @@ public class InputData {
protected boolean filterStringValid;
protected String filterString;
protected String dataSource;
- protected String profile;
protected String accessor;
protected String resolver;
protected String fragmenter;
@@ -95,7 +94,6 @@ public class InputData {
this.filterStringValid = copy.filterStringValid;
this.filterString = copy.filterString;
this.dataSource = copy.dataSource;
- this.profile = copy.profile;
this.accessor = copy.accessor;
this.resolver = copy.resolver;
this.fragmenter = copy.fragmenter;
@@ -248,15 +246,6 @@ public class InputData {
}
/**
- * Returns the profile name.
- *
- * @return name of profile
- */
- public String getProfile() {
- return profile;
- }
-
- /**
* Returns the ClassName for the java class that was defined as Accessor.
*
* @return class name for Accessor
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2f8392e6/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
index 2d2b53e..97a297e 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
@@ -56,10 +56,8 @@ import org.apache.hawq.pxf.api.Metadata;
import org.apache.hawq.pxf.api.io.DataType;
import org.apache.hawq.pxf.api.utilities.ColumnDescriptor;
import org.apache.hawq.pxf.api.utilities.InputData;
-import org.apache.hawq.pxf.api.utilities.ProfilesConf;
import org.apache.hawq.pxf.plugins.hdfs.utilities.HdfsUtilities;
import org.apache.hawq.pxf.plugins.hive.utilities.HiveUtilities;
-import org.apache.hawq.pxf.service.ProfileFactory;
/**
* Fragmenter class for HIVE tables. <br>
@@ -78,10 +76,10 @@ public class HiveDataFragmenter extends Fragmenter {
private static final Log LOG = LogFactory.getLog(HiveDataFragmenter.class);
private static final short ALL_PARTS = -1;
- public static final String HIVE_UD_DELIM = "!HUDD!";
- public static final String HIVE_1_PART_DELIM = "!H1PD!";
- public static final String HIVE_PARTITIONS_DELIM = "!HPAD!";
- public static final String HIVE_NO_PART_TBL = "!HNPT!";
+ static final String HIVE_UD_DELIM = "!HUDD!";
+ static final String HIVE_1_PART_DELIM = "!H1PD!";
+ static final String HIVE_PARTITIONS_DELIM = "!HPAD!";
+ static final String HIVE_NO_PART_TBL = "!HNPT!";
static final String HIVE_API_EQ = " = ";
static final String HIVE_API_LT = " < ";
@@ -103,6 +101,36 @@ public class HiveDataFragmenter extends Fragmenter {
private Map<String, String> partitionkeyTypes = new HashMap<>();
/**
+ * A Hive table unit - means a subset of the HIVE table, where we can say
+ * that for all files in this subset, they all have the same InputFormat and
+ * Serde. For a partitioned table the HiveTableUnit will be one partition
+ * and for an unpartitioned table, the HiveTableUnit will be the whole table
+ */
+ class HiveTablePartition {
+ StorageDescriptor storageDesc;
+ Properties properties;
+ Partition partition;
+ List<FieldSchema> partitionKeys;
+ String tableName;
+
+ HiveTablePartition(StorageDescriptor storageDesc,
+ Properties properties, Partition partition,
+ List<FieldSchema> partitionKeys, String tableName) {
+ this.storageDesc = storageDesc;
+ this.properties = properties;
+ this.partition = partition;
+ this.partitionKeys = partitionKeys;
+ this.tableName = tableName;
+ }
+
+ @Override
+ public String toString() {
+ return "table - " + tableName
+ + ((partition == null) ? "" : ", partition - " + partition);
+ }
+ }
+
+ /**
* Constructs a HiveDataFragmenter object.
*
* @param inputData all input parameters coming from the client
@@ -281,19 +309,6 @@ public class HiveDataFragmenter extends Fragmenter {
throws Exception {
InputFormat<?, ?> fformat = makeInputFormat(
tablePartition.storageDesc.getInputFormat(), jobConf);
- String profile = null;
- if (inputData.getProfile() != null) {
- // evaluate optimal profile based on file format if profile was explicitly specified in url
- // if user passed accessor+fragmenter+resolver - use them
- profile = ProfileFactory.get(fformat);
- }
- String fragmenterForProfile = null;
- if (profile != null) {
- fragmenterForProfile = ProfilesConf.getProfilePluginsMap(profile).get("X-GP-FRAGMENTER");
- } else {
- fragmenterForProfile = inputData.getFragmenter();
- }
-
FileInputFormat.setInputPaths(jobConf, new Path(
tablePartition.storageDesc.getLocation()));
@@ -312,11 +327,57 @@ public class HiveDataFragmenter extends Fragmenter {
byte[] locationInfo = HdfsUtilities.prepareFragmentMetadata(fsp);
Fragment fragment = new Fragment(filepath, hosts, locationInfo,
- HiveUtilities.makeUserData(fragmenterForProfile, tablePartition, filterInFragmenter), profile);
+ makeUserData(tablePartition));
fragments.add(fragment);
}
}
+ /* Turns a Properties class into a string */
+ private String serializeProperties(Properties props) throws Exception {
+ ByteArrayOutputStream outStream = new ByteArrayOutputStream();
+ props.store(outStream, ""/* comments */);
+ return outStream.toString();
+ }
+
+ /* Turns the partition keys into a string */
+ String serializePartitionKeys(HiveTablePartition partData) throws Exception {
+ if (partData.partition == null) /*
+ * this is a simple hive table - there
+ * are no partitions
+ */{
+ return HIVE_NO_PART_TBL;
+ }
+
+ StringBuilder partitionKeys = new StringBuilder();
+ String prefix = "";
+ ListIterator<String> valsIter = partData.partition.getValues().listIterator();
+ ListIterator<FieldSchema> keysIter = partData.partitionKeys.listIterator();
+ while (valsIter.hasNext() && keysIter.hasNext()) {
+ FieldSchema key = keysIter.next();
+ String name = key.getName();
+ String type = key.getType();
+ String val = valsIter.next();
+ String oneLevel = prefix + name + HIVE_1_PART_DELIM + type
+ + HIVE_1_PART_DELIM + val;
+ partitionKeys.append(oneLevel);
+ prefix = HIVE_PARTITIONS_DELIM;
+ }
+
+ return partitionKeys.toString();
+ }
+
+ byte[] makeUserData(HiveTablePartition partData) throws Exception {
+ String inputFormatName = partData.storageDesc.getInputFormat();
+ String serdeName = partData.storageDesc.getSerdeInfo().getSerializationLib();
+ String propertiesString = serializeProperties(partData.properties);
+ String partitionKeys = serializePartitionKeys(partData);
+ String userData = inputFormatName + HIVE_UD_DELIM + serdeName
+ + HIVE_UD_DELIM + propertiesString + HIVE_UD_DELIM
+ + partitionKeys + HIVE_UD_DELIM + filterInFragmenter;
+
+ return userData.getBytes();
+ }
+
/*
* Build filter string for HiveMetaStoreClient.listPartitionsByFilter API
* method.
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2f8392e6/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveInputFormatFragmenter.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveInputFormatFragmenter.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveInputFormatFragmenter.java
index 051a246..b6a6041 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveInputFormatFragmenter.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveInputFormatFragmenter.java
@@ -55,6 +55,14 @@ import java.util.List;
*/
public class HiveInputFormatFragmenter extends HiveDataFragmenter {
private static final Log LOG = LogFactory.getLog(HiveInputFormatFragmenter.class);
+
+ static final String STR_RC_FILE_INPUT_FORMAT = "org.apache.hadoop.hive.ql.io.RCFileInputFormat";
+ static final String STR_TEXT_FILE_INPUT_FORMAT = "org.apache.hadoop.mapred.TextInputFormat";
+ static final String STR_ORC_FILE_INPUT_FORMAT = "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat";
+ static final String STR_COLUMNAR_SERDE = "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe";
+ static final String STR_LAZY_BINARY_COLUMNAR_SERDE = "org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe";
+ static final String STR_LAZY_SIMPLE_SERDE = "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe";
+ static final String STR_ORC_SERDE = "org.apache.hadoop.hive.ql.io.orc.OrcSerde";
private static final int EXPECTED_NUM_OF_TOKS = 3;
public static final int TOK_SERDE = 0;
public static final int TOK_KEYS = 1;
@@ -156,6 +164,69 @@ public class HiveInputFormatFragmenter extends HiveDataFragmenter {
}
+ /*
+ * Validates that partition format corresponds to PXF supported formats and
+ * transforms the class name to an enumeration for writing it to the
+ * accessors on other PXF instances.
+ */
+ private String assertFileType(String className, HiveTablePartition partData)
+ throws Exception {
+ switch (className) {
+ case STR_RC_FILE_INPUT_FORMAT:
+ return PXF_HIVE_INPUT_FORMATS.RC_FILE_INPUT_FORMAT.name();
+ case STR_TEXT_FILE_INPUT_FORMAT:
+ return PXF_HIVE_INPUT_FORMATS.TEXT_FILE_INPUT_FORMAT.name();
+ case STR_ORC_FILE_INPUT_FORMAT:
+ return PXF_HIVE_INPUT_FORMATS.ORC_FILE_INPUT_FORMAT.name();
+ default:
+ throw new IllegalArgumentException(
+ "HiveInputFormatFragmenter does not yet support "
+ + className
+ + " for "
+ + partData
+ + ". Supported InputFormat are "
+ + Arrays.toString(PXF_HIVE_INPUT_FORMATS.values()));
+ }
+ }
+
+ /*
+ * Validates that partition serde corresponds to PXF supported serdes and
+ * transforms the class name to an enumeration for writing it to the
+ * resolvers on other PXF instances.
+ */
+ private String assertSerde(String className, HiveTablePartition partData)
+ throws Exception {
+ switch (className) {
+ case STR_COLUMNAR_SERDE:
+ return PXF_HIVE_SERDES.COLUMNAR_SERDE.name();
+ case STR_LAZY_BINARY_COLUMNAR_SERDE:
+ return PXF_HIVE_SERDES.LAZY_BINARY_COLUMNAR_SERDE.name();
+ case STR_LAZY_SIMPLE_SERDE:
+ return PXF_HIVE_SERDES.LAZY_SIMPLE_SERDE.name();
+ case STR_ORC_SERDE:
+ return PXF_HIVE_SERDES.ORC_SERDE.name();
+ default:
+ throw new UnsupportedTypeException(
+ "HiveInputFormatFragmenter does not yet support "
+ + className + " for " + partData
+ + ". Supported serializers are: "
+ + Arrays.toString(PXF_HIVE_SERDES.values()));
+ }
+ }
+
+ @Override
+ byte[] makeUserData(HiveTablePartition partData) throws Exception {
+ String inputFormatName = partData.storageDesc.getInputFormat();
+ String serdeName = partData.storageDesc.getSerdeInfo().getSerializationLib();
+ String partitionKeys = serializePartitionKeys(partData);
+
+ assertFileType(inputFormatName, partData);
+ String userData = assertSerde(serdeName, partData) + HIVE_UD_DELIM
+ + partitionKeys + HIVE_UD_DELIM + filterInFragmenter;
+
+ return userData.getBytes();
+ }
+
/**
* Returns statistics for Hive table. Currently it's not implemented.
*/
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2f8392e6/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveTablePartition.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveTablePartition.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveTablePartition.java
deleted file mode 100644
index 3905fef..0000000
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveTablePartition.java
+++ /dev/null
@@ -1,56 +0,0 @@
-package org.apache.hawq.pxf.plugins.hive;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import java.util.List;
-import java.util.Properties;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-
-/**
- * A Hive table unit - means a subset of the HIVE table, where we can say
- * that for all files in this subset, they all have the same InputFormat and
- * Serde. For a partitioned table the HiveTableUnit will be one partition
- * and for an unpartitioned table, the HiveTableUnit will be the whole table
- */
-public class HiveTablePartition {
- public StorageDescriptor storageDesc;
- public Properties properties;
- public Partition partition;
- public List<FieldSchema> partitionKeys;
- public String tableName;
-
- HiveTablePartition(StorageDescriptor storageDesc,
- Properties properties, Partition partition,
- List<FieldSchema> partitionKeys, String tableName) {
- this.storageDesc = storageDesc;
- this.properties = properties;
- this.partition = partition;
- this.partitionKeys = partitionKeys;
- this.tableName = tableName;
- }
-
- @Override
- public String toString() {
- return "table - " + tableName
- + ((partition == null) ? "" : ", partition - " + partition);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2f8392e6/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java
index 575b129..e18fcd8 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilities.java
@@ -20,12 +20,9 @@ package org.apache.hawq.pxf.plugins.hive.utilities;
*/
-import java.io.ByteArrayOutputStream;
import java.util.Arrays;
import java.util.List;
import java.util.ArrayList;
-import java.util.ListIterator;
-import java.util.Properties;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -36,16 +33,10 @@ import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hawq.pxf.api.Fragmenter;
import org.apache.hawq.pxf.api.Metadata;
import org.apache.hawq.pxf.api.UnsupportedTypeException;
import org.apache.hawq.pxf.api.utilities.EnumHawqType;
import org.apache.hawq.pxf.api.io.DataType;
-import org.apache.hawq.pxf.plugins.hive.HiveDataFragmenter;
-import org.apache.hawq.pxf.plugins.hive.HiveInputFormatFragmenter;
-import org.apache.hawq.pxf.plugins.hive.HiveTablePartition;
-import org.apache.hawq.pxf.plugins.hive.HiveInputFormatFragmenter.PXF_HIVE_INPUT_FORMATS;
-import org.apache.hawq.pxf.plugins.hive.HiveInputFormatFragmenter.PXF_HIVE_SERDES;
/**
* Class containing helper functions connecting
@@ -61,14 +52,6 @@ public class HiveUtilities {
*/
private static final String HIVE_DEFAULT_DBNAME = "default";
- static final String STR_RC_FILE_INPUT_FORMAT = "org.apache.hadoop.hive.ql.io.RCFileInputFormat";
- static final String STR_TEXT_FILE_INPUT_FORMAT = "org.apache.hadoop.mapred.TextInputFormat";
- static final String STR_ORC_FILE_INPUT_FORMAT = "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat";
- static final String STR_COLUMNAR_SERDE = "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe";
- static final String STR_LAZY_BINARY_COLUMNAR_SERDE = "org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe";
- static final String STR_LAZY_SIMPLE_SERDE = "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe";
- static final String STR_ORC_SERDE = "org.apache.hadoop.hive.ql.io.orc.OrcSerde";
-
/**
* Initializes the HiveMetaStoreClient
* Uses classpath configuration files to locate the MetaStore
@@ -343,116 +326,4 @@ public class HiveUtilities {
break;
}
}
-
- /* Turns a Properties class into a string */
- private static String serializeProperties(Properties props) throws Exception {
- ByteArrayOutputStream outStream = new ByteArrayOutputStream();
- props.store(outStream, ""/* comments */);
- return outStream.toString();
- }
-
- /*
- * Validates that partition format corresponds to PXF supported formats and
- * transforms the class name to an enumeration for writing it to the
- * accessors on other PXF instances.
- */
- private static String assertFileType(String className, HiveTablePartition partData)
- throws Exception {
- switch (className) {
- case STR_RC_FILE_INPUT_FORMAT:
- return PXF_HIVE_INPUT_FORMATS.RC_FILE_INPUT_FORMAT.name();
- case STR_TEXT_FILE_INPUT_FORMAT:
- return PXF_HIVE_INPUT_FORMATS.TEXT_FILE_INPUT_FORMAT.name();
- case STR_ORC_FILE_INPUT_FORMAT:
- return PXF_HIVE_INPUT_FORMATS.ORC_FILE_INPUT_FORMAT.name();
- default:
- throw new IllegalArgumentException(
- "HiveInputFormatFragmenter does not yet support "
- + className
- + " for "
- + partData
- + ". Supported InputFormat are "
- + Arrays.toString(PXF_HIVE_INPUT_FORMATS.values()));
- }
- }
-
- /*
- * Validates that partition serde corresponds to PXF supported serdes and
- * transforms the class name to an enumeration for writing it to the
- * resolvers on other PXF instances.
- */
- private static String assertSerde(String className, HiveTablePartition partData)
- throws Exception {
- switch (className) {
- case STR_COLUMNAR_SERDE:
- return PXF_HIVE_SERDES.COLUMNAR_SERDE.name();
- case STR_LAZY_BINARY_COLUMNAR_SERDE:
- return PXF_HIVE_SERDES.LAZY_BINARY_COLUMNAR_SERDE.name();
- case STR_LAZY_SIMPLE_SERDE:
- return PXF_HIVE_SERDES.LAZY_SIMPLE_SERDE.name();
- case STR_ORC_SERDE:
- return PXF_HIVE_SERDES.ORC_SERDE.name();
- default:
- throw new UnsupportedTypeException(
- "HiveInputFormatFragmenter does not yet support "
- + className + " for " + partData
- + ". Supported serializers are: "
- + Arrays.toString(PXF_HIVE_SERDES.values()));
- }
- }
-
-
- /* Turns the partition keys into a string */
- public static String serializePartitionKeys(HiveTablePartition partData) throws Exception {
- if (partData.partition == null) /*
- * this is a simple hive table - there
- * are no partitions
- */{
- return HiveDataFragmenter.HIVE_NO_PART_TBL;
- }
-
- StringBuilder partitionKeys = new StringBuilder();
- String prefix = "";
- ListIterator<String> valsIter = partData.partition.getValues().listIterator();
- ListIterator<FieldSchema> keysIter = partData.partitionKeys.listIterator();
- while (valsIter.hasNext() && keysIter.hasNext()) {
- FieldSchema key = keysIter.next();
- String name = key.getName();
- String type = key.getType();
- String val = valsIter.next();
- String oneLevel = prefix + name + HiveDataFragmenter.HIVE_1_PART_DELIM + type
- + HiveDataFragmenter.HIVE_1_PART_DELIM + val;
- partitionKeys.append(oneLevel);
- prefix = HiveDataFragmenter.HIVE_PARTITIONS_DELIM;
- }
-
- return partitionKeys.toString();
- }
-
- public static byte[] makeUserData(String fragmenterClassName, HiveTablePartition partData, boolean filterInFragmenter) throws Exception {
-
- String userData = null;
-
- if (fragmenterClassName == null) {
- throw new IllegalArgumentException("No fragmenter provided.");
- }
-
- if (fragmenterClassName.equals("org.apache.hawq.pxf.plugins.hive.HiveInputFormatFragmenter")) {
- String inputFormatName = partData.storageDesc.getInputFormat();
- String serdeName = partData.storageDesc.getSerdeInfo().getSerializationLib();
- String partitionKeys = serializePartitionKeys(partData);
- assertFileType(inputFormatName, partData);
- userData = assertSerde(serdeName, partData) + HiveDataFragmenter.HIVE_UD_DELIM
- + partitionKeys + HiveDataFragmenter.HIVE_UD_DELIM + filterInFragmenter;
- } else {
- String inputFormatName = partData.storageDesc.getInputFormat();
- String serdeName = partData.storageDesc.getSerdeInfo().getSerializationLib();
- String propertiesString = serializeProperties(partData.properties);
- String partitionKeys = serializePartitionKeys(partData);
- userData = inputFormatName + HiveDataFragmenter.HIVE_UD_DELIM + serdeName
- + HiveDataFragmenter.HIVE_UD_DELIM + propertiesString + HiveDataFragmenter.HIVE_UD_DELIM
- + partitionKeys + HiveDataFragmenter.HIVE_UD_DELIM + filterInFragmenter;
- }
- return userData.getBytes();
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2f8392e6/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/ProfileFactory.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/ProfileFactory.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/ProfileFactory.java
deleted file mode 100644
index fc5ed0f..0000000
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/ProfileFactory.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package org.apache.hawq.pxf.service;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
-import org.apache.hadoop.mapred.InputFormat;
-
-public class ProfileFactory {
-
- private static final String HIVE_TEXT_PROFILE = "HiveText";
- private static final String HIVE_RC_PROFILE = "HiveRC";
- private static final String HIVE_ORC_PROFILE = "HiveORC";
-
- public static String get(InputFormat inputFormat) throws Exception {
- String profileName = null;
- // TODO: Uncomment in process of HAWQ-1228 implementation
- //if (inputFormat instanceof TextInputFormat) {
- // profileName = HIVE_TEXT_PROFILE;
- //} else if (inputFormat instanceof RCFileInputFormat) {
- // profileName = HIVE_RC_PROFILE;
- /*} else */if (inputFormat instanceof OrcInputFormat) {
- profileName = HIVE_ORC_PROFILE;
- }
-
- return profileName;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2f8392e6/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
index a0e63ce..a250b18 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
@@ -50,6 +50,7 @@ public class ProtocolData extends InputData {
protected OutputFormat outputFormat;
protected int port;
protected String host;
+ protected String profile;
protected String token;
// statistics parameters
protected int statsMaxFragments;
[2/2] incubator-hawq git commit: Revert "HAWQ-1179. Call Bridge api
with profile value read from Fragmenter call."
Posted by od...@apache.org.
Revert "HAWQ-1179. Call Bridge api with profile value read from Fragmenter call."
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/90a69266
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/90a69266
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/90a69266
Branch: refs/heads/master
Commit: 90a69266eb7af6b61fa32414e11299ae535c98cc
Parents: 2f8392e
Author: Oleksandr Diachenko <od...@pivotal.io>
Authored: Thu Dec 22 01:55:59 2016 -0800
Committer: Oleksandr Diachenko <od...@pivotal.io>
Committed: Thu Dec 22 01:55:59 2016 -0800
----------------------------------------------------------------------
src/backend/access/external/hd_work_mgr.c | 21 +-
src/backend/access/external/pxfheaders.c | 14 +-
src/backend/access/external/pxfmasterapi.c | 9 -
src/backend/access/external/pxfuriparser.c | 156 +-----
.../access/external/test/hd_work_mgr_test.c | 112 ----
.../access/external/test/pxfuriparser_test.c | 558 +------------------
src/bin/gpfusion/gpbridgeapi.c | 11 -
src/include/access/pxfmasterapi.h | 1 -
src/include/access/pxfuriparser.h | 3 -
9 files changed, 44 insertions(+), 841 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/90a69266/src/backend/access/external/hd_work_mgr.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/hd_work_mgr.c b/src/backend/access/external/hd_work_mgr.c
index 520b5de..6829de5 100644
--- a/src/backend/access/external/hd_work_mgr.c
+++ b/src/backend/access/external/hd_work_mgr.c
@@ -53,7 +53,6 @@ typedef struct sAllocatedDataFragment
char *source_name; /* source name */
char *fragment_md; /* fragment meta data */
char *user_data; /* additional user data */
- char *profile; /* recommended profile to work with fragment */
} AllocatedDataFragment;
/*
@@ -714,7 +713,6 @@ create_allocated_fragment(DataFragment *fragment)
allocated->source_name = pstrdup(fragment->source_name);
allocated->fragment_md = (fragment->fragment_md) ? pstrdup(fragment->fragment_md) : NULL;
allocated->user_data = (fragment->user_data) ? pstrdup(fragment->user_data) : NULL;
- allocated->profile = (fragment->profile) ? pstrdup(fragment->profile) : NULL;
return allocated;
}
@@ -784,22 +782,12 @@ make_allocation_output_string(List *segment_fragments)
appendStringInfo(&fragment_str, "%d", frag->index);
appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
if (frag->fragment_md)
- {
appendStringInfo(&fragment_str, "%s", frag->fragment_md);
- }
-
- appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
if (frag->user_data)
{
+ appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
appendStringInfo(&fragment_str, "%s", frag->user_data);
}
- appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
- if (frag->profile)
- {
- appendStringInfo(&fragment_str, "%s", frag->profile);
- }
- appendStringInfoChar(&fragment_str, SEGWORK_IN_PAIR_DELIM);
-
fragment_size = strlen(fragment_str.data);
appendStringInfo(&segwork, "%d", fragment_size);
@@ -829,8 +817,6 @@ free_allocated_frags(List *segment_fragments)
pfree(frag->fragment_md);
if (frag->user_data)
pfree(frag->user_data);
- if (frag->profile)
- pfree(frag->profile);
pfree(frag);
}
list_free(segment_fragments);
@@ -870,11 +856,6 @@ print_fragment_list(List *fragments)
{
appendStringInfo(&log_str, "user data: %s\n", frag->user_data);
}
-
- if (frag->profile)
- {
- appendStringInfo(&log_str, "profile: %s\n", frag->profile);
- }
}
elog(FRAGDEBUG, "%s", log_str.data);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/90a69266/src/backend/access/external/pxfheaders.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/pxfheaders.c b/src/backend/access/external/pxfheaders.c
index 2381044..fe7cd6a 100644
--- a/src/backend/access/external/pxfheaders.c
+++ b/src/backend/access/external/pxfheaders.c
@@ -27,12 +27,12 @@
#include "catalog/pg_exttable.h"
#include "access/pxfheaders.h"
#include "access/pxffilters.h"
-#include "utils/formatting.h"
#include "utils/guc.h"
static void add_alignment_size_httpheader(CHURL_HEADERS headers);
static void add_tuple_desc_httpheader(CHURL_HEADERS headers, Relation rel);
static void add_location_options_httpheader(CHURL_HEADERS headers, GPHDUri *gphduri);
+static char* prepend_x_gp(const char* key);
static void add_delegation_token_headers(CHURL_HEADERS headers, PxfInputData *inputData);
static void add_remote_credentials(CHURL_HEADERS headers);
static void add_projection_desc_httpheader(CHURL_HEADERS headers, ProjectionInfo *projInfo, List *qualsAttributes);
@@ -303,12 +303,22 @@ static void add_location_options_httpheader(CHURL_HEADERS headers, GPHDUri *gphd
foreach(option, gphduri->options)
{
OptionData *data = (OptionData*)lfirst(option);
- char *x_gp_key = normalize_key_name(data->key);
+ char *x_gp_key = prepend_x_gp(data->key);
churl_headers_append(headers, x_gp_key, data->value);
pfree(x_gp_key);
}
}
+/* Full name of the HEADER KEY expected by the PXF service */
+static char* prepend_x_gp(const char* key)
+{
+ StringInfoData formatter;
+ initStringInfo(&formatter);
+ appendStringInfo(&formatter, "X-GP-%s", key);
+
+ return formatter.data;
+}
+
/*
* The function will add delegation token headers.
* Each token information piece will be serialized as HEX string.
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/90a69266/src/backend/access/external/pxfmasterapi.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/pxfmasterapi.c b/src/backend/access/external/pxfmasterapi.c
index 4ee90ff..833218a 100644
--- a/src/backend/access/external/pxfmasterapi.c
+++ b/src/backend/access/external/pxfmasterapi.c
@@ -358,11 +358,6 @@ parse_get_fragments_response(List *fragments, StringInfo rest_buf)
if (js_user_data)
fragment->user_data = pstrdup(json_object_get_string(js_user_data));
- /* 5. profile - recommended profile to work with fragment */
- struct json_object *js_profile = json_object_object_get(js_fragment, "profile");
- if (js_profile)
- fragment->profile = pstrdup(json_object_get_string(js_profile));
-
/*
* HD-2547:
* Ignore fragment if it doesn't contain any host locations,
@@ -405,10 +400,6 @@ void free_fragment(DataFragment *fragment)
if (fragment->user_data)
pfree(fragment->user_data);
-
- if (fragment->profile)
- pfree(fragment->profile);
-
pfree(fragment);
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/90a69266/src/backend/access/external/pxfuriparser.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/pxfuriparser.c b/src/backend/access/external/pxfuriparser.c
index 17d5803..9e83714 100644
--- a/src/backend/access/external/pxfuriparser.c
+++ b/src/backend/access/external/pxfuriparser.c
@@ -33,7 +33,7 @@ static void GPHDUri_parse_protocol(GPHDUri *uri, char **cursor);
static void GPHDUri_parse_authority(GPHDUri *uri, char **cursor);
static void GPHDUri_parse_data(GPHDUri *uri, char **cursor);
static void GPHDUri_parse_options(GPHDUri *uri, char **cursor);
-static List* GPHDUri_parse_option(char* pair, GPHDUri *uri);
+static List* GPHDUri_parse_option(char* pair, List* options, const char* uri);
static void GPHDUri_free_options(GPHDUri *uri);
static void GPHDUri_parse_segwork(GPHDUri *uri, const char *uri_str);
static List* GPHDUri_parse_fragment(char* fragment, List* fragments);
@@ -41,7 +41,6 @@ static void GPHDUri_free_fragments(GPHDUri *uri);
static void GPHDUri_debug_print_options(GPHDUri *uri);
static void GPHDUri_debug_print_segwork(GPHDUri *uri);
static void GPHDUri_fetch_authority_from_ha_nn(GPHDUri *uri, char *nameservice);
-char* normalize_key_name(const char* key);
/* parseGPHDUri
*
@@ -120,8 +119,6 @@ freeGPHDUri(GPHDUri *uri)
pfree(uri->host);
pfree(uri->port);
pfree(uri->data);
- if (uri->profile)
- pfree(uri->profile);
GPHDUri_free_options(uri);
if (uri->ha_nodes)
@@ -136,8 +133,6 @@ freeGPHDUriForMetadata(GPHDUri *uri)
pfree(uri->host);
pfree(uri->port);
- if (uri->profile)
- pfree(uri->profile);
pfree(uri);
}
@@ -477,7 +472,7 @@ GPHDUri_parse_options(GPHDUri *uri, char **cursor)
pair;
pair = strtok_r(NULL, sep, &strtok_context))
{
- uri->options = GPHDUri_parse_option(pair, uri);
+ uri->options = GPHDUri_parse_option(pair, uri->options, uri->uri);
}
pfree(dup);
@@ -489,7 +484,7 @@ GPHDUri_parse_options(GPHDUri *uri, char **cursor)
* to OptionData object (key and value).
*/
static List*
-GPHDUri_parse_option(char* pair, GPHDUri *uri)
+GPHDUri_parse_option(char* pair, List* options, const char* uri)
{
char *sep;
@@ -503,40 +498,33 @@ GPHDUri_parse_option(char* pair, GPHDUri *uri)
if (sep == NULL) {
ereport(ERROR,
(errcode(ERRCODE_SYNTAX_ERROR),
- errmsg("Invalid URI %s: option '%s' missing '='", uri->uri, pair)));
+ errmsg("Invalid URI %s: option '%s' missing '='", uri, pair)));
}
if (strchr(sep + 1, '=') != NULL) {
ereport(ERROR,
(errcode(ERRCODE_SYNTAX_ERROR),
- errmsg("Invalid URI %s: option '%s' contains duplicate '='", uri->uri, pair)));
+ errmsg("Invalid URI %s: option '%s' contains duplicate '='", uri, pair)));
}
key_len = sep - pair;
if (key_len == 0) {
ereport(ERROR,
(errcode(ERRCODE_SYNTAX_ERROR),
- errmsg("Invalid URI %s: option '%s' missing key before '='", uri->uri, pair)));
+ errmsg("Invalid URI %s: option '%s' missing key before '='", uri, pair)));
}
value_len = pair_len - key_len + 1;
if (value_len == EMPTY_VALUE_LEN) {
ereport(ERROR,
(errcode(ERRCODE_SYNTAX_ERROR),
- errmsg("Invalid URI %s: option '%s' missing value after '='", uri->uri, pair)));
+ errmsg("Invalid URI %s: option '%s' missing value after '='", uri, pair)));
}
option_data->key = pnstrdup(pair,key_len);
option_data->value = pnstrdup(sep + 1, value_len);
- char *x_gp_key = normalize_key_name(option_data->key);
- if (strcmp(x_gp_key, "X-GP-PROFILE") == 0)
- {
- uri->profile = pstrdup(option_data->value);
- }
- pfree(x_gp_key);
-
- return lappend(uri->options, option_data);
+ return lappend(options, option_data);
}
/*
@@ -574,11 +562,6 @@ GPHDUri_debug_print(GPHDUri *uri)
uri->port,
uri->data);
- if (uri->profile)
- {
- elog(NOTICE, "Profile: %s", uri->profile);
- }
-
GPHDUri_debug_print_options(uri);
GPHDUri_debug_print_segwork(uri);
}
@@ -628,8 +611,6 @@ GPHDUri_debug_print_segwork(GPHDUri *uri)
data->fragment_md ? data->fragment_md : "NULL");
if (data->user_data)
appendStringInfo(&fragment_data, ", user data : %s", data->user_data);
- if (data->profile)
- appendStringInfo(&fragment_data, ", profile : %s", data->profile);
elog(NOTICE, "%s", fragment_data.data);
++count;
resetStringInfo(&fragment_data);
@@ -690,123 +671,66 @@ GPHDUri_parse_segwork(GPHDUri *uri, const char *uri_str)
static List*
GPHDUri_parse_fragment(char* fragment, List* fragments)
{
- if (!fragment)
- {
- ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("internal error in pxfuriparser.c:GPHDUri_parse_fragment.fragment string is null.")));
- }
char *dup_frag = pstrdup(fragment);
char *value_start;
char *value_end;
+ bool has_user_data = false;
- StringInfoData authority_formatter;
+ StringInfoData formatter;
FragmentData* fragment_data;
fragment_data = palloc0(sizeof(FragmentData));
- initStringInfo(&authority_formatter);
+ initStringInfo(&formatter);
value_start = dup_frag;
-
/* expect ip */
value_end = strchr(value_start, segwork_separator);
- if (value_end == NULL)
- {
- ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("internal error in pxfuriparser.c:GPHDUri_parse_fragment. Fragment string is invalid.")));
- }
+ Assert(value_end != NULL);
*value_end = '\0';
- appendStringInfo(&authority_formatter, "%s:", value_start);
+ appendStringInfo(&formatter, "%s:", value_start);
value_start = value_end + 1;
-
/* expect port */
value_end = strchr(value_start, segwork_separator);
- if (value_end == NULL)
- {
- ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("internal error in pxfuriparser.c:GPHDUri_parse_fragment. Fragment string is invalid.")));
- }
+ Assert(value_end != NULL);
*value_end = '\0';
- appendStringInfo(&authority_formatter, "%s", value_start);
- fragment_data->authority = pstrdup(authority_formatter.data);
- pfree(authority_formatter.data);
+ appendStringInfo(&formatter, "%s", value_start);
+ fragment_data->authority = formatter.data;
value_start = value_end + 1;
-
/* expect source name */
value_end = strchr(value_start, segwork_separator);
- if (value_end == NULL)
- {
- ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("internal error in pxfuriparser.c:GPHDUri_parse_fragment. Fragment string is invalid.")));
- }
+ Assert(value_end != NULL);
*value_end = '\0';
fragment_data->source_name = pstrdup(value_start);
value_start = value_end + 1;
-
/* expect index */
value_end = strchr(value_start, segwork_separator);
- if (value_end == NULL)
- {
- ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("internal error in pxfuriparser.c:GPHDUri_parse_fragment. Fragment string is invalid.")));
- }
+ Assert(value_end != NULL);
*value_end = '\0';
fragment_data->index = pstrdup(value_start);
value_start = value_end + 1;
-
/* expect fragment metadata */
Assert(value_start);
- value_end = strchr(value_start, segwork_separator);
- if (value_end == NULL)
- {
- ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("internal error in pxfuriparser.c:GPHDUri_parse_fragment. Fragment string is invalid.")));
- }
- *value_end = '\0';
- fragment_data->fragment_md = pstrdup(value_start);
- value_start = value_end + 1;
- /* expect user data */
- Assert(value_start);
+ /* check for user data */
value_end = strchr(value_start, segwork_separator);
- if (value_end == NULL)
+ if (value_end != NULL)
{
- ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("internal error in pxfuriparser.c:GPHDUri_parse_fragment. Fragment string is invalid.")));
+ has_user_data = true;
+ *value_end = '\0';
}
- *value_end = '\0';
- fragment_data->user_data = pstrdup(value_start);
- value_start = value_end + 1;
+ fragment_data->fragment_md = pstrdup(value_start);
- /* expect for profile */
- Assert(value_start);
- value_end = strchr(value_start, segwork_separator);
- if (value_end == NULL)
+ /* read user data */
+ if (has_user_data)
{
- ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("internal error in pxfuriparser.c:GPHDUri_parse_fragment. Fragment string is invalid.")));
+ fragment_data->user_data = pstrdup(value_end + 1);
}
- *value_end = '\0';
- if (strlen(value_start) > 0)
- fragment_data->profile = pstrdup(value_start);
return lappend(fragments, fragment_data);
}
/*
- * Free fragment data
- */
-static void
-GPHDUri_free_fragment(FragmentData *data)
-{
- if (data->authority)
- pfree(data->authority);
- if (data->fragment_md)
- pfree(data->fragment_md);
- if (data->index)
- pfree(data->index);
- if (data->profile)
- pfree(data->profile);
- if (data->source_name)
- pfree(data->source_name);
- if (data->user_data)
- pfree(data->user_data);
- pfree(data);
-}
-
-/*
* Free fragments list
*/
static void
@@ -817,7 +741,10 @@ GPHDUri_free_fragments(GPHDUri *uri)
foreach(fragment, uri->fragments)
{
FragmentData *data = (FragmentData*)lfirst(fragment);
- GPHDUri_free_fragment(data);
+ pfree(data->authority);
+ pfree(data->index);
+ pfree(data->source_name);
+ pfree(data);
}
list_free(uri->fragments);
uri->fragments = NIL;
@@ -896,26 +823,3 @@ bool RelationIsExternalPxfReadOnly(Relation rel, StringInfo location)
return false;
}
-
-/*
- * Full name of the HEADER KEY expected by the PXF service
- * Converts input string to upper case and prepends "X-GP-" string
- *
- */
-char* normalize_key_name(const char* key)
-{
- if (!key || strlen(key) == 0)
- {
- ereport(ERROR,
- (errcode(ERRCODE_INTERNAL_ERROR),
- errmsg("internal error in pxfheaders.c:normalize_key_name. Parameter key is null or empty.")));
- }
-
- StringInfoData formatter;
- initStringInfo(&formatter);
- char* upperCasedKey = str_toupper(pstrdup(key), strlen(key));
- appendStringInfo(&formatter, "X-GP-%s", upperCasedKey);
- pfree(upperCasedKey);
-
- return formatter.data;
-}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/90a69266/src/backend/access/external/test/hd_work_mgr_test.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/test/hd_work_mgr_test.c b/src/backend/access/external/test/hd_work_mgr_test.c
index 759b461..c39b2ac 100644
--- a/src/backend/access/external/test/hd_work_mgr_test.c
+++ b/src/backend/access/external/test/hd_work_mgr_test.c
@@ -28,124 +28,12 @@
#include "hd_work_mgr_allocate_fragments_to_datanodes_test.c"
#include "hd_work_mgr_distribute_work_2_gp_segments_test.c"
-/*
- * Serialize output string when all fields of AllocatedDataFragment are passed
- */
-void
-test__make_allocation_output_string(void **state)
-{
-
- List *segment_fragments = NIL;
- AllocatedDataFragment* frag = palloc0(sizeof(AllocatedDataFragment));
-
- frag->index = 42;
- frag->host = "HOST";
- frag->rest_port = 1312;
- frag->source_name = "TABLE_NAME";
- frag->fragment_md = "FRAGMENT_METADATA";
- frag->user_data = "USER_DATA";
- frag->profile = "PROFILE";
-
- segment_fragments = lappend(segment_fragments, frag);
- char *output_str = make_allocation_output_string(segment_fragments);
-
- assert_string_equal(output_str, "segwork=60@HOST@1312@TABLE_NAME@42@FRAGMENT_METADATA@USER_DATA@PROFILE@");
-
- pfree(frag);
- pfree(output_str);
- list_free(segment_fragments);
-}
-
-/*
- * Serialize output string when profile is empty
- */
-void
-test__make_allocation_output_string__empty_profile(void **state)
-{
-
- List *segment_fragments = NIL;
- AllocatedDataFragment* frag = palloc0(sizeof(AllocatedDataFragment));
-
- frag->index = 42;
- frag->host = "HOST";
- frag->rest_port = 1312;
- frag->source_name = "TABLE_NAME";
- frag->fragment_md = "FRAGMENT_METADATA";
- frag->user_data = "USER_DATA";
-
- segment_fragments = lappend(segment_fragments, frag);
- char *output_str = make_allocation_output_string(segment_fragments);
-
- assert_string_equal(output_str, "segwork=53@HOST@1312@TABLE_NAME@42@FRAGMENT_METADATA@USER_DATA@@");
-
- pfree(frag);
- pfree(output_str);
- list_free(segment_fragments);
-}
-
-/*
- * Serialize output string when user data is empty
- */
-void
-test__make_allocation_output_string__empty_user_data(void **state)
-{
-
- List *segment_fragments = NIL;
- AllocatedDataFragment* frag = palloc0(sizeof(AllocatedDataFragment));
-
- frag->index = 42;
- frag->host = "HOST";
- frag->rest_port = 1312;
- frag->source_name = "TABLE_NAME";
- frag->fragment_md = "FRAGMENT_METADATA";
- frag->profile = "PROFILE";
-
- segment_fragments = lappend(segment_fragments, frag);
- char *output_str = make_allocation_output_string(segment_fragments);
-
- assert_string_equal(output_str, "segwork=51@HOST@1312@TABLE_NAME@42@FRAGMENT_METADATA@@PROFILE@");
-
- pfree(frag);
- pfree(output_str);
- list_free(segment_fragments);
-}
-
-/*
- * Serialize output string when profile and user data are empty
- */
-void
-test__make_allocation_output_string__empty_user_data_profile(void **state)
-{
-
- List *segment_fragments = NIL;
- AllocatedDataFragment* frag = palloc0(sizeof(AllocatedDataFragment));
-
- frag->index = 42;
- frag->host = "HOST";
- frag->rest_port = 1312;
- frag->source_name = "TABLE_NAME";
- frag->fragment_md = "FRAGMENT_METADATA";
-
- segment_fragments = lappend(segment_fragments, frag);
- char *output_str = make_allocation_output_string(segment_fragments);
-
- assert_string_equal(output_str, "segwork=44@HOST@1312@TABLE_NAME@42@FRAGMENT_METADATA@@@");
-
- pfree(frag);
- pfree(output_str);
- list_free(segment_fragments);
-}
-
int
main(int argc, char* argv[])
{
cmockery_parse_arguments(argc, argv);
const UnitTest tests[] = {
- unit_test(test__make_allocation_output_string),
- unit_test(test__make_allocation_output_string__empty_profile),
- unit_test(test__make_allocation_output_string__empty_user_data),
- unit_test(test__make_allocation_output_string__empty_user_data_profile),
unit_test(test__do_segment_clustering_by_host__10SegmentsOn3Hosts),
unit_test(test__get_dn_processing_load),
unit_test(test__create_allocated_fragment__NoUserData),
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/90a69266/src/backend/access/external/test/pxfuriparser_test.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/test/pxfuriparser_test.c b/src/backend/access/external/test/pxfuriparser_test.c
index 1912d2b..b97cef5 100644
--- a/src/backend/access/external/test/pxfuriparser_test.c
+++ b/src/backend/access/external/test/pxfuriparser_test.c
@@ -72,7 +72,6 @@ test__parseGPHDUri__ValidURI(void **state)
assert_string_equal(option->value, "SomeAnalyzer");
assert_true(parsed->fragments == NULL);
- assert_true(parsed->profile == NULL);
freeGPHDUri(parsed);
}
@@ -181,7 +180,6 @@ test__parseGPHDUri__NegativeTestNoProtocol(void **state)
assert_true(edata->sqlerrcode == ERRCODE_SYNTAX_ERROR);
assert_true(edata->elevel == ERROR);
assert_string_equal(edata->message, "Invalid URI pxf:/1.2.3.4:5678/some/path/and/table.tbl?FRAGMENTER=HdfsDataFragmenter");
- elog_dismiss(INFO);
return;
}
PG_END_TRY();
@@ -212,7 +210,6 @@ test__parseGPHDUri__NegativeTestNoOptions(void **state)
assert_true(edata->sqlerrcode == ERRCODE_SYNTAX_ERROR);
assert_true(edata->elevel == ERROR);
assert_string_equal(edata->message, "Invalid URI pxf://1.2.3.4:5678/some/path/and/table.tbl: missing options section");
- elog_dismiss(INFO);
return;
}
PG_END_TRY();
@@ -243,7 +240,6 @@ test__parseGPHDUri__NegativeTestMissingEqual(void **state)
assert_true(edata->sqlerrcode == ERRCODE_SYNTAX_ERROR);
assert_true(edata->elevel == ERROR);
assert_string_equal(edata->message, "Invalid URI pxf://1.2.3.4:5678/some/path/and/table.tbl?FRAGMENTER: option 'FRAGMENTER' missing '='");
- elog_dismiss(INFO);
return;
}
PG_END_TRY();
@@ -274,7 +270,6 @@ test__parseGPHDUri__NegativeTestDuplicateEquals(void **state)
assert_true(edata->sqlerrcode == ERRCODE_SYNTAX_ERROR);
assert_true(edata->elevel == ERROR);
assert_string_equal(edata->message, "Invalid URI pxf://1.2.3.4:5678/some/path/and/table.tbl?FRAGMENTER=HdfsDataFragmenter=DuplicateFragmenter: option 'FRAGMENTER=HdfsDataFragmenter=DuplicateFragmenter' contains duplicate '='");
- elog_dismiss(INFO);
return;
}
PG_END_TRY();
@@ -305,7 +300,6 @@ test__parseGPHDUri__NegativeTestMissingKey(void **state)
assert_true(edata->sqlerrcode == ERRCODE_SYNTAX_ERROR);
assert_true(edata->elevel == ERROR);
assert_string_equal(edata->message, "Invalid URI pxf://1.2.3.4:5678/some/path/and/table.tbl?=HdfsDataFragmenter: option '=HdfsDataFragmenter' missing key before '='");
- elog_dismiss(INFO);
return;
}
PG_END_TRY();
@@ -336,7 +330,6 @@ test__parseGPHDUri__NegativeTestMissingValue(void **state)
assert_true(edata->sqlerrcode == ERRCODE_SYNTAX_ERROR);
assert_true(edata->elevel == ERROR);
assert_string_equal(edata->message, "Invalid URI pxf://1.2.3.4:5678/some/path/and/table.tbl?FRAGMENTER=: option 'FRAGMENTER=' missing value after '='");
- elog_dismiss(INFO);
return;
}
PG_END_TRY();
@@ -355,7 +348,6 @@ test__GPHDUri_verify_no_duplicate_options__ValidURI(void **state)
/* Setting the test -- code omitted -- */
GPHDUri* parsed = parseGPHDUri(valid_uri);
GPHDUri_verify_no_duplicate_options(parsed);
- assert_string_equal(parsed->profile, "a");
freeGPHDUri(parsed);
}
@@ -383,7 +375,6 @@ test__GPHDUri_verify_no_duplicate_options__NegativeTestDuplicateOpts(void **stat
assert_true(edata->sqlerrcode == ERRCODE_SYNTAX_ERROR);
assert_true(edata->elevel == ERROR);
assert_string_equal(edata->message, "Invalid URI pxf://1.2.3.4:5678/some/path/and/table.tbl?Profile=a&Analyzer=b&PROFILE=c: Duplicate option(s): PROFILE");
- elog_dismiss(INFO);
return;
}
PG_END_TRY();
@@ -433,7 +424,6 @@ test__GPHDUri_verify_core_options_exist__NegativeTestMissingCoreOpts(void **stat
assert_true(edata->elevel == ERROR);
assert_string_equal(edata->message, "Invalid URI pxf://1.2.3.4:5678/some/path/and/table.tbl?FRAGMENTER=a: PROFILE or ACCESSOR and RESOLVER option(s) missing");
list_free(coreOptions);
- elog_dismiss(INFO);
return;
}
PG_END_TRY();
@@ -441,541 +431,9 @@ test__GPHDUri_verify_core_options_exist__NegativeTestMissingCoreOpts(void **stat
assert_true(false);
}
-/*
- * Test GPHDUri_parse_fragment when fragment string is valid and all parameters are passed
- */
-void
-test__GPHDUri_parse_fragment__ValidFragment(void **state) {
-
- char* fragment = "HOST@REST_PORT@TABLE_NAME@INDEX@FRAGMENT_METADATA@USER_DATA@PROFILE@";
-
- List *fragments = NIL;
-
- fragments = GPHDUri_parse_fragment(fragment, fragments);
-
- ListCell *fragment_cell = list_head(fragments);
- FragmentData *fragment_data = (FragmentData*) lfirst(fragment_cell);
-
- assert_string_equal(fragment_data->authority, "HOST:REST_PORT");
- assert_string_equal(fragment_data->fragment_md, "FRAGMENT_METADATA");
- assert_string_equal(fragment_data->index, "INDEX");
- assert_string_equal(fragment_data->profile, "PROFILE");
- assert_string_equal(fragment_data->source_name, "TABLE_NAME");
- assert_string_equal(fragment_data->user_data, "USER_DATA");
-
- GPHDUri_free_fragment(fragment_data);
- list_free(fragments);
-
-}
-
-/*
- * Test GPHDUri_parse_fragment when fragment string doesn't have profile
- */
-void
-test__GPHDUri_parse_fragment__EmptyProfile(void **state) {
- char* fragment = "HOST@REST_PORT@TABLE_NAME@INDEX@FRAGMENT_METADATA@USER_DATA@@";
-
- List *fragments = NIL;
-
- fragments = GPHDUri_parse_fragment(fragment, fragments);
-
- ListCell *fragment_cell = list_head(fragments);
- FragmentData *fragment_data = (FragmentData*) lfirst(fragment_cell);
-
- assert_string_equal(fragment_data->authority, "HOST:REST_PORT");
- assert_string_equal(fragment_data->fragment_md, "FRAGMENT_METADATA");
- assert_string_equal(fragment_data->index, "INDEX");
- assert_true(!fragment_data->profile);
- assert_string_equal(fragment_data->source_name, "TABLE_NAME");
- assert_string_equal(fragment_data->user_data, "USER_DATA");
-
- GPHDUri_free_fragment(fragment_data);
- list_free(fragments);
-
-}
-
-/*
- * Test GPHDUri_parse_fragment when fragment string is null
- */
-void
-test__GPHDUri_parse_fragment__NullFragment(void **state) {
-
- char* fragment = NULL;
-
- List *fragments = NIL;
-
- StringInfo err_msg = makeStringInfo();
- appendStringInfo(err_msg, "internal error in pxfuriparser.c:GPHDUri_parse_fragment.fragment string is null.");
-
- /* Expect error */
- PG_TRY();
- {
- /* This will throw a ereport(ERROR).*/
- fragments = GPHDUri_parse_fragment(fragment, fragments);
- }
- PG_CATCH();
- {
- CurrentMemoryContext = 1;
- ErrorData *edata = CopyErrorData();
-
- /* Validate the type of expected error */
- assert_true(edata->sqlerrcode == ERRCODE_INTERNAL_ERROR);
- assert_true(edata->elevel == ERROR);
- assert_string_equal(edata->message, err_msg->data);
-
- list_free(fragments);
- pfree(err_msg->data);
- pfree(err_msg);
- elog_dismiss(INFO);
- return;
- }
- PG_END_TRY();
-
- /* should not reach here*/
- assert_true(false);
-
-}
-
-
-/*
- * Test GPHDUri_parse_fragment when fragment string is empty
- */
-void
-test__GPHDUri_parse_fragment__EmptyString(void **state) {
-
- char* fragment = "";
-
- List *fragments = NIL;
-
- StringInfo err_msg = makeStringInfo();
- appendStringInfo(err_msg, "internal error in pxfuriparser.c:GPHDUri_parse_fragment. Fragment string is invalid.");
-
- /* Expect error */
- PG_TRY();
- {
- /* This will throw a ereport(ERROR).*/
- fragments = GPHDUri_parse_fragment(fragment, fragments);
- }
- PG_CATCH();
- {
- CurrentMemoryContext = 1;
- ErrorData *edata = CopyErrorData();
-
- /* Validate the type of expected error */
- assert_true(edata->sqlerrcode == ERRCODE_INTERNAL_ERROR);
- assert_true(edata->elevel == ERROR);
- assert_string_equal(edata->message, err_msg->data);
-
- list_free(fragments);
- pfree(err_msg->data);
- pfree(err_msg);
- elog_dismiss(INFO);
- return;
- }
- PG_END_TRY();
-
- /* should not reach here*/
- assert_true(false);
-
-}
-
-/*
- * Test GPHDUri_parse_fragment when fragment string has less tokens then expected
- */
-void
-test__GPHDUri_parse_fragment__MissingIpHost(void **state) {
-
- char* fragment = "@";
-
- List *fragments = NIL;
-
- StringInfo err_msg = makeStringInfo();
- appendStringInfo(err_msg, "internal error in pxfuriparser.c:GPHDUri_parse_fragment. Fragment string is invalid.");
-
- /* Expect error */
- PG_TRY();
- {
- /* This will throw a ereport(ERROR).*/
- fragments = GPHDUri_parse_fragment(fragment, fragments);
- }
- PG_CATCH();
- {
- CurrentMemoryContext = 1;
- ErrorData *edata = CopyErrorData();
-
- /* Validate the type of expected error */
- assert_true(edata->sqlerrcode == ERRCODE_INTERNAL_ERROR);
- assert_true(edata->elevel == ERROR);
- assert_string_equal(edata->message, err_msg->data);
-
- list_free(fragments);
- pfree(err_msg->data);
- pfree(err_msg);
- elog_dismiss(INFO);
- return;
- }
- PG_END_TRY();
-
- /* should not reach here*/
- assert_true(false);
-
-}
-
-/*
- * Test GPHDUri_parse_fragment when fragment string has less tokens then expected
- */
-void
-test__GPHDUri_parse_fragment__MissingPort(void **state) {
-
- char* fragment = "@HOST@";
-
- List *fragments = NIL;
-
- StringInfo err_msg = makeStringInfo();
- appendStringInfo(err_msg, "internal error in pxfuriparser.c:GPHDUri_parse_fragment. Fragment string is invalid.");
-
- /* Expect error */
- PG_TRY();
- {
- /* This will throw a ereport(ERROR).*/
- fragments = GPHDUri_parse_fragment(fragment, fragments);
- }
- PG_CATCH();
- {
- CurrentMemoryContext = 1;
- ErrorData *edata = CopyErrorData();
-
- /* Validate the type of expected error */
- assert_true(edata->sqlerrcode == ERRCODE_INTERNAL_ERROR);
- assert_true(edata->elevel == ERROR);
- assert_string_equal(edata->message, err_msg->data);
-
- list_free(fragments);
- pfree(err_msg->data);
- pfree(err_msg);
-
- elog_dismiss(INFO);
- return;
- }
- PG_END_TRY();
-
- /* should not reach here*/
- assert_true(false);
-
-}
-
-/*
- * Test GPHDUri_parse_fragment when fragment string has less tokens then expected
- */
-void
-test__GPHDUri_parse_fragment__MissingSourceName(void **state) {
-
- char* fragment = "@HOST@PORT@";
-
- List *fragments = NIL;
-
- StringInfo err_msg = makeStringInfo();
- appendStringInfo(err_msg, "internal error in pxfuriparser.c:GPHDUri_parse_fragment. Fragment string is invalid.");
-
- /* Expect error */
- PG_TRY();
- {
- /* This will throw a ereport(ERROR).*/
- fragments = GPHDUri_parse_fragment(fragment, fragments);
- }
- PG_CATCH();
- {
- CurrentMemoryContext = 1;
- ErrorData *edata = CopyErrorData();
-
- /* Validate the type of expected error */
- assert_true(edata->sqlerrcode == ERRCODE_INTERNAL_ERROR);
- assert_true(edata->elevel == ERROR);
- assert_string_equal(edata->message, err_msg->data);
-
- list_free(fragments);
- pfree(err_msg->data);
- pfree(err_msg);
- elog_dismiss(INFO);
- return;
- }
- PG_END_TRY();
-
- /* should not reach here*/
- assert_true(false);
-
-}
-
-/*
- * Test GPHDUri_parse_fragment when fragment string has less tokens then expected
- */
-void
-test__GPHDUri_parse_fragment__MissingIndex(void **state) {
-
-
- char* fragment = "@HOST@PORT@SOURCE_NAME@";
-
- List *fragments = NIL;
-
- StringInfo err_msg = makeStringInfo();
- appendStringInfo(err_msg, "internal error in pxfuriparser.c:GPHDUri_parse_fragment. Fragment string is invalid.");
-
- /* Expect error */
- PG_TRY();
- {
- /* This will throw a ereport(ERROR).*/
- fragments = GPHDUri_parse_fragment(fragment, fragments);
- }
- PG_CATCH();
- {
- CurrentMemoryContext = 1;
- ErrorData *edata = CopyErrorData();
-
- /* Validate the type of expected error */
- assert_true(edata->sqlerrcode == ERRCODE_INTERNAL_ERROR);
- assert_true(edata->elevel == ERROR);
- assert_string_equal(edata->message, err_msg->data);
-
- list_free(fragments);
- pfree(err_msg->data);
- pfree(err_msg);
- elog_dismiss(INFO);
- return;
- }
- PG_END_TRY();
-
- /* should not reach here*/
- assert_true(false);
-
-}
-
-
-/*
- * Test GPHDUri_parse_fragment when fragment string has less tokens then expected
- */
-void
-test__GPHDUri_parse_fragment__MissingFragmentMetadata(void **state) {
-
- char* fragment = "@HOST@PORT@SOURCE_NAME@42@";
-
- List *fragments = NIL;
-
- StringInfo err_msg = makeStringInfo();
- appendStringInfo(err_msg, "internal error in pxfuriparser.c:GPHDUri_parse_fragment. Fragment string is invalid.");
-
- /* Expect error */
- PG_TRY();
- {
- /* This will throw a ereport(ERROR).*/
- fragments = GPHDUri_parse_fragment(fragment, fragments);
- }
- PG_CATCH();
- {
- CurrentMemoryContext = 1;
- ErrorData *edata = CopyErrorData();
-
- /* Validate the type of expected error */
- assert_true(edata->sqlerrcode == ERRCODE_INTERNAL_ERROR);
- assert_true(edata->elevel == ERROR);
- assert_string_equal(edata->message, err_msg->data);
-
- list_free(fragments);
- pfree(err_msg->data);
- pfree(err_msg);
- elog_dismiss(INFO);
- return;
- }
- PG_END_TRY();
-
- /* should not reach here*/
- assert_true(false);
-
-}
-
-/*
- * Test GPHDUri_parse_fragment when fragment string has less tokens then expected
- */
-void
-test__GPHDUri_parse_fragment__MissingUserData(void **state) {
-
- char* fragment = "HOST@REST_PORT@TABLE_NAME@INDEX@FRAGMENT_METADATA@";
-
- List *fragments = NIL;
-
- StringInfo err_msg = makeStringInfo();
- appendStringInfo(err_msg, "internal error in pxfuriparser.c:GPHDUri_parse_fragment. Fragment string is invalid.");
-
- /* Expect error */
- PG_TRY();
- {
- /* This will throw a ereport(ERROR).*/
- fragments = GPHDUri_parse_fragment(fragment, fragments);
- }
- PG_CATCH();
- {
- CurrentMemoryContext = 1;
- ErrorData *edata = CopyErrorData();
-
- /* Validate the type of expected error */
- assert_true(edata->sqlerrcode == ERRCODE_INTERNAL_ERROR);
- assert_true(edata->elevel == ERROR);
- assert_string_equal(edata->message, err_msg->data);
-
- list_free(fragments);
- pfree(err_msg->data);
- pfree(err_msg);
- elog_dismiss(INFO);
- return;
- }
- PG_END_TRY();
-
- /* should not reach here*/
- assert_true(false);
-
-}
-
-
-/*
- * Test GPHDUri_parse_fragment when fragment string has less tokens then expected
- */
-void
-test__GPHDUri_parse_fragment__MissingProfile(void **state) {
-
- char* fragment = "HOST@REST_PORT@TABLE_NAME@INDEX@FRAGMENT_METADATA@USER_METADATA@";
-
- List *fragments = NIL;
-
- StringInfo err_msg = makeStringInfo();
- appendStringInfo(err_msg, "internal error in pxfuriparser.c:GPHDUri_parse_fragment. Fragment string is invalid.");
-
- /* Expect error */
- PG_TRY();
- {
- /* This will throw a ereport(ERROR).*/
- fragments = GPHDUri_parse_fragment(fragment, fragments);
- }
- PG_CATCH();
- {
- CurrentMemoryContext = 1;
- ErrorData *edata = CopyErrorData();
-
- /* Validate the type of expected error */
- assert_true(edata->sqlerrcode == ERRCODE_INTERNAL_ERROR);
- assert_true(edata->elevel == ERROR);
- assert_string_equal(edata->message, err_msg->data);
-
- list_free(fragments);
- pfree(err_msg->data);
- pfree(err_msg);
- elog_dismiss(INFO);
- return;
- }
- PG_END_TRY();
-
- /* should not reach here*/
- assert_true(false);
-
-}
-
-
-void
-test__normalize_key_name_Positive(void **state)
-{
- char *input_key = strdup("mIxEdCaSeVaLuE");
- char *normalized_key = normalize_key_name(input_key);
- assert_string_equal(normalized_key, "X-GP-MIXEDCASEVALUE");
-
- pfree(input_key);
- pfree(normalized_key);
-}
-
-void
-test__normalize_key_name_PositiveUpperCase(void **state)
-{
- char *input_key = strdup("ALREADY_UPPER_CASE");
- char *normalized_key = normalize_key_name(input_key);
- assert_string_equal(normalized_key, "X-GP-ALREADY_UPPER_CASE");
-
- pfree(input_key);
- pfree(normalized_key);
-}
-
-void
-test__normalize_key_name_Negative__key_is_null(void **state)
-{
- char *input_key = NULL;
-
- StringInfo err_msg = makeStringInfo();
- appendStringInfo(err_msg, "internal error in pxfheaders.c:normalize_key_name. Parameter key is null or empty.");
-
- /* Expect error */
- PG_TRY();
- {
- /* This will throw a ereport(ERROR).*/
- char *normalized_key = normalize_key_name(input_key);
- }
- PG_CATCH();
- {
- CurrentMemoryContext = 1;
- ErrorData *edata = CopyErrorData();
-
- /* Validate the type of expected error */
- assert_true(edata->sqlerrcode == ERRCODE_INTERNAL_ERROR);
- assert_true(edata->elevel == ERROR);
- assert_string_equal(edata->message, err_msg->data);
-
- pfree(err_msg->data);
- pfree(err_msg);
-
- return;
- }
- PG_END_TRY();
-
- /* should not reach here*/
- assert_true(false);
-
-}
-
-void
-test__normalize_key_name_Negative__key_is_empty(void **state)
-{
- char *input_key = "";
-
- StringInfo err_msg = makeStringInfo();
- appendStringInfo(err_msg, "internal error in pxfheaders.c:normalize_key_name. Parameter key is null or empty.");
-
- /* Expect error */
- PG_TRY();
- {
- /* This will throw a ereport(ERROR).*/
- char *normalized_key = normalize_key_name(input_key);
- }
- PG_CATCH();
- {
- CurrentMemoryContext = 1;
- ErrorData *edata = CopyErrorData();
-
- /* Validate the type of expected error */
- assert_true(edata->sqlerrcode == ERRCODE_INTERNAL_ERROR);
- assert_true(edata->elevel == ERROR);
- assert_string_equal(edata->message, err_msg->data);
-
- pfree(err_msg->data);
- pfree(err_msg);
-
- return;
- }
- PG_END_TRY();
-
- /* should not reach here*/
- assert_true(false);
-
-}
-
int
main(int argc, char* argv[])
{
-
cmockery_parse_arguments(argc, argv);
const UnitTest tests[] = {
@@ -991,21 +449,7 @@ main(int argc, char* argv[])
unit_test(test__GPHDUri_verify_no_duplicate_options__ValidURI),
unit_test(test__GPHDUri_verify_no_duplicate_options__NegativeTestDuplicateOpts),
unit_test(test__GPHDUri_verify_core_options_exist__ValidURI),
- unit_test(test__GPHDUri_verify_core_options_exist__NegativeTestMissingCoreOpts),
- unit_test(test__GPHDUri_parse_fragment__EmptyProfile),
- unit_test(test__GPHDUri_parse_fragment__ValidFragment),
- unit_test(test__GPHDUri_parse_fragment__EmptyString),
- unit_test(test__GPHDUri_parse_fragment__MissingIpHost),
- unit_test(test__GPHDUri_parse_fragment__MissingPort),
- unit_test(test__GPHDUri_parse_fragment__MissingSourceName),
- unit_test(test__GPHDUri_parse_fragment__MissingIndex),
- unit_test(test__GPHDUri_parse_fragment__MissingFragmentMetadata),
- unit_test(test__GPHDUri_parse_fragment__MissingUserData),
- unit_test(test__GPHDUri_parse_fragment__MissingProfile),
- unit_test(test__normalize_key_name_Positive),
- unit_test(test__normalize_key_name_PositiveUpperCase),
- unit_test(test__normalize_key_name_Negative__key_is_null),
- unit_test(test__normalize_key_name_Negative__key_is_empty)
+ unit_test(test__GPHDUri_verify_core_options_exist__NegativeTestMissingCoreOpts)
};
return run_tests(tests);
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/90a69266/src/bin/gpfusion/gpbridgeapi.c
----------------------------------------------------------------------
diff --git a/src/bin/gpfusion/gpbridgeapi.c b/src/bin/gpfusion/gpbridgeapi.c
index b524df8..2751b1b 100644
--- a/src/bin/gpfusion/gpbridgeapi.c
+++ b/src/bin/gpfusion/gpbridgeapi.c
@@ -226,17 +226,6 @@ void set_current_fragment_headers(gphadoop_context* context)
churl_headers_remove(context->churl_headers, "X-GP-FRAGMENT-USER-DATA", true);
}
- /* if current fragment has optimal profile set it*/
- if (frag_data->profile)
- {
- churl_headers_override(context->churl_headers, "X-GP-PROFILE", frag_data->profile);
- } else if (context->gphd_uri->profile)
- {
- /* if current fragment doesn't have any optimal profile, set to use profile from url */
- churl_headers_override(context->churl_headers, "X-GP-PROFILE", context->gphd_uri->profile);
- }
- /* if there is no profile passed in url, we expect to have accessor+fragmenter+resolver so no action needed by this point */
-
}
void gpbridge_import_start(PG_FUNCTION_ARGS)
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/90a69266/src/include/access/pxfmasterapi.h
----------------------------------------------------------------------
diff --git a/src/include/access/pxfmasterapi.h b/src/include/access/pxfmasterapi.h
index b487b41..4b9ecd9 100644
--- a/src/include/access/pxfmasterapi.h
+++ b/src/include/access/pxfmasterapi.h
@@ -56,7 +56,6 @@ typedef struct sDataFragment
List *replicas;
char *fragment_md; /* fragment meta data (start, end, length, etc.) */
char *user_data;
- char *profile;
} DataFragment;
/*
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/90a69266/src/include/access/pxfuriparser.h
----------------------------------------------------------------------
diff --git a/src/include/access/pxfuriparser.h b/src/include/access/pxfuriparser.h
index ac614c5..d050325 100644
--- a/src/include/access/pxfuriparser.h
+++ b/src/include/access/pxfuriparser.h
@@ -46,7 +46,6 @@ typedef struct FragmentData
char *source_name;
char *fragment_md;
char *user_data;
- char *profile;
} FragmentData;
typedef struct OptionData
@@ -67,7 +66,6 @@ typedef struct GPHDUri
char *host; /* host name str */
char *port; /* port number as string */
char *data; /* data location (path) */
- char *profile; /* profile option */
List *fragments; /* list of FragmentData */
/* options */
@@ -90,6 +88,5 @@ int GPHDUri_get_value_for_opt(GPHDUri *uri, char *key, char **val, bool emit_e
bool RelationIsExternalPxfReadOnly(Relation rel, StringInfo location);
void GPHDUri_verify_no_duplicate_options(GPHDUri *uri);
void GPHDUri_verify_core_options_exist(GPHDUri *uri, List *coreOptions);
-char* normalize_key_name(const char* key);
#endif // _PXF_URIPARSER_H_