You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by od...@apache.org on 2016/05/03 02:33:29 UTC
[13/13] incubator-hawq git commit: HAWQ-703. Serialize HCatalog
Complex Types to plain text (as Hive profile).
HAWQ-703. Serialize HCatalog Complex Types to plain text (as Hive profile).
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b1db3991
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b1db3991
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b1db3991
Branch: refs/heads/HAWQ-703
Commit: b1db3991cc9be1536a17b93953d81f29c8dfa323
Parents: 6717e7d b357dbc
Author: Oleksandr Diachenko <od...@pivotal.io>
Authored: Mon May 2 13:00:44 2016 -0700
Committer: Oleksandr Diachenko <od...@pivotal.io>
Committed: Mon May 2 17:33:09 2016 -0700
----------------------------------------------------------------------
depends/libyarn/src/CMakeLists.txt | 2 +-
.../libyarn/src/libyarnclient/LibYarnClient.cpp | 6 +-
pxf/Makefile | 5 +-
pxf/build.gradle | 5 +
.../org/apache/hawq/pxf/api/package-info.java | 23 ++
.../hawq/pxf/api/utilities/Utilities.java | 11 +-
.../hawq/pxf/plugins/hbase/package-info.java | 23 ++
.../hawq/pxf/plugins/hdfs/AvroResolver.java | 4 +-
.../hawq/pxf/plugins/hdfs/package-info.java | 23 ++
.../hawq/pxf/plugins/hive/package-info.java | 23 ++
.../hive/utilities/EnumHiveToHawqType.java | 2 +-
.../hive/utilities/HiveUtilitiesTest.java | 8 +-
.../pxf/service/MetadataResponseFormatter.java | 1 +
.../apache/hawq/pxf/service/package-info.java | 23 ++
.../pxf/service/utilities/ProtocolData.java | 2 +-
src/backend/executor/execMain.c | 2 +-
.../gpopt/translate/CTranslatorUtils.cpp | 10 +-
.../communication/rmcomm_RM2RMSEG.c | 13 +-
src/backend/utils/misc/etc/gpcheck.cnf | 48 ++-
src/bin/psql/describe.c | 2 +
tools/bin/gpcheck | 372 ++++++++++++++++++-
tools/bin/gppylib/gpcheckutil.py | 17 +-
tools/bin/hawqpylib/hawqlib.py | 18 +
tools/doc/gpcheck_help | 44 ++-
tools/sbin/gpcheck_hostdump | 50 ++-
25 files changed, 684 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b1db3991/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/EnumHiveToHawqType.java
----------------------------------------------------------------------
diff --cc pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/EnumHiveToHawqType.java
index 13d402c,0000000..a9ff877
mode 100644,000000..100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/EnumHiveToHawqType.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/EnumHiveToHawqType.java
@@@ -1,112 -1,0 +1,112 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hawq.pxf.plugins.hive.utilities;
+
+import org.apache.hawq.pxf.api.utilities.EnumHawqType;
+import org.apache.hawq.pxf.api.UnsupportedTypeException;
+
+/**
+ *
+ * Hive types, which are supported by plugin, mapped to HAWQ's types
+ * @see EnumHawqType
+ */
+public enum EnumHiveToHawqType {
+
+ TinyintType("tinyint", EnumHawqType.Int2Type),
+ SmallintType("smallint", EnumHawqType.Int2Type),
+ IntType("int", EnumHawqType.Int4Type),
+ BigintType("bigint", EnumHawqType.Int8Type),
+ BooleanType("boolean", EnumHawqType.BoolType),
+ FloatType("float", EnumHawqType.Float4Type),
+ DoubleType("double", EnumHawqType.Float8Type),
+ StringType("string", EnumHawqType.TextType),
+ BinaryType("binary", EnumHawqType.ByteaType),
+ TimestampType("timestamp", EnumHawqType.TimestampType),
+ DateType("date", EnumHawqType.DateType),
+ DecimalType("decimal", EnumHawqType.NumericType, "[(,)]"),
+ VarcharType("varchar", EnumHawqType.VarcharType, "[(,)]"),
+ CharType("char", EnumHawqType.BpcharType, "[(,)]"),
+ ArrayType("array", EnumHawqType.TextType, "[<,>]"),
+ MapType("map", EnumHawqType.TextType, "[<,>]"),
+ StructType("struct", EnumHawqType.TextType, "[<,>]"),
+ UnionType("uniontype", EnumHawqType.TextType, "[<,>]");
+
+ private String typeName;
+ private EnumHawqType hawqType;
+ private String splitExpression;
+
+ EnumHiveToHawqType(String typeName, EnumHawqType hawqType) {
+ this.typeName = typeName;
+ this.hawqType = hawqType;
+ }
+
+ EnumHiveToHawqType(String typeName, EnumHawqType hawqType, String splitExpression) {
+ this(typeName, hawqType);
+ this.splitExpression = splitExpression;
+ }
+
+ /**
+ *
+ * @return name of type
+ */
+ public String getTypeName() {
+ return this.typeName;
+ }
+
+ /**
+ *
+ * @return corresponding HAWQ type
+ */
+ public EnumHawqType getHawqType() {
+ return this.hawqType;
+ }
+
+ /**
+ *
+ * @return split by expression
+ */
+ public String getSplitExpression() {
+ return this.splitExpression;
+ }
+
+ /**
+ * Returns Hive to HAWQ type mapping entry for given Hive type
+ *
+ * @param hiveType full Hive type with modifiers, for example - decimal(10, 0), char(5), binary, array<string>, map<string,float> etc
+ * @return corresponding Hive to HAWQ type mapping entry
+ * @throws UnsupportedTypeException if there is no corresponding HAWQ type
+ */
+ public static EnumHiveToHawqType getHiveToHawqType(String hiveType) {
+ for (EnumHiveToHawqType t : values()) {
+ String hiveTypeName = hiveType;
+ if (t.getSplitExpression() != null) {
+ String[] tokens = hiveType.split(t.getSplitExpression());
+ hiveTypeName = tokens[0];
+ }
+
- if (t.getTypeName().equals(hiveTypeName)) {
++ if (t.getTypeName().toLowerCase().equals(hiveTypeName.toLowerCase())) {
+ return t;
+ }
+ }
+ throw new UnsupportedTypeException("Unable to map Hive's type: "
+ + hiveType + " to HAWQ's type");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b1db3991/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
----------------------------------------------------------------------
diff --cc pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
index 1054d0d,466dedb..e9b024a
--- a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
+++ b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
@@@ -54,12 -54,6 +54,13 @@@ public class HiveUtilitiesTest
{"char(40)", "bpchar", "40"},
};
+ static String[][] complexTypes = {
- {"array<string>", "text"},
- {"map<string,float>", "text"},
- {"struct<street:string,city:string,state:string,zip:int>", "text"},
++ {"ArraY<string>", "text"},
++ {"MaP<stRing, float>", "text"},
++ {"Struct<street:string, city:string, state:string, zip:int>", "text"},
++ {"UnionType<array<string>, string,int>", "text"}
+ };
+
@Test
public void mapHiveTypeUnsupported() throws Exception {
@@@ -157,37 -152,6 +158,38 @@@
}
@Test
+ public void mapHiveTypeInvalidModifiers() throws Exception {
+ String badHiveType = "decimal(abc, xyz)";
+ hiveColumn = new FieldSchema("numericColumn", badHiveType, null);
+ try {
+ HiveUtilities.mapHiveType(hiveColumn);
+ fail("should fail with bad modifiers error");
+ } catch (UnsupportedTypeException e) {
+ String errorMsg = "HAWQ does not support type " + badHiveType + " (Field numericColumn), modifiers should be integers";
+ assertEquals(errorMsg, e.getMessage());
+ }
+ }
+
+ @Test
+ public void mapHiveTypeComplex() throws Exception {
+ /*
+ * array<dataType> -> text
+ * map<keyDataType, valueDataType> -> text
+ * struct<fieldName1:dataType, ..., fieldNameN:dataType> -> text
++ * uniontype<...> -> text
+ */
+ for (String[] line: complexTypes) {
+ String hiveType = line[0];
+ String expectedType = line[1];
+ hiveColumn = new FieldSchema("field" + hiveType, hiveType, null);
+ Metadata.Field result = HiveUtilities.mapHiveType(hiveColumn);
+ assertEquals("field" + hiveType, result.getName());
+ assertEquals(expectedType, result.getType().getTypeName());
+ assertNull(result.getModifiers());
+ }
+ }
+
+ @Test
public void parseTableQualifiedNameNoDbName() throws Exception {
String name = "orphan";
tblDesc = HiveUtilities.extractTableFromName(name);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b1db3991/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b1db3991/src/bin/psql/describe.c
----------------------------------------------------------------------