You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2018/01/26 11:20:51 UTC
[25/33] kylin git commit: KYLIN-3167, fulfil datatype's precision
KYLIN-3167, fulfil datatype's precision
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1d825a1e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1d825a1e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1d825a1e
Branch: refs/heads/sync
Commit: 1d825a1e1869ac18b4c494cb838da60af9a841df
Parents: 5bdf723
Author: Cheng Wang <ch...@kyligence.io>
Authored: Tue Jan 16 14:55:36 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 19:02:20 2018 +0800
----------------------------------------------------------------------
.../kylin/source/hive/BeelineHiveClient.java | 27 +++++++++--
.../source/hive/BeelineHIveClientTest.java | 51 ++++++++++++++++++++
2 files changed, 74 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1d825a1e/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 314402c..747b1bb 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -114,7 +114,7 @@ public class BeelineHiveClient implements IHiveClient {
}
return count;
}
-
+
@Override
public void executeHQL(String hql) throws IOException {
throw new UnsupportedOperationException();
@@ -132,7 +132,11 @@ public class BeelineHiveClient implements IHiveClient {
List<HiveTableMeta.HiveTableColumnMeta> allColumns = Lists.newArrayList();
while (columns.next()) {
- allColumns.add(new HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6), columns.getString(12)));
+ String columnName = columns.getString(4);
+ String dataType = columns.getString(6);
+ String comment = columns.getString(12);
+ dataType = considerDataTypePrecision(dataType, columns.getString(7), columns.getString(9));
+ allColumns.add(new HiveTableMeta.HiveTableColumnMeta(columnName, dataType, comment));
}
builder.setAllColumns(allColumns);
DBUtils.closeQuietly(columns);
@@ -143,6 +147,19 @@ public class BeelineHiveClient implements IHiveClient {
return builder.createHiveTableMeta();
}
+ public static String considerDataTypePrecision(String dataType, String precision, String scale) {
+ if ("VARCHAR".equalsIgnoreCase(dataType) || "CHAR".equalsIgnoreCase(dataType)) {
+ if (null != precision)
+ dataType = new StringBuilder(dataType).append("(").append(precision).append(")").toString();
+ }
+ if ("DECIMAL".equalsIgnoreCase(dataType) || "NUMERIC".equalsIgnoreCase(dataType)) {
+ if (precision != null && scale != null)
+ dataType = new StringBuilder(dataType).append("(").append(precision).append(",").append(scale)
+ .append(")").toString();
+ }
+ return dataType;
+ }
+
private void extractHiveTableMeta(ResultSet resultSet, HiveTableMetaBuilder builder) throws SQLException {
while (resultSet.next()) {
@@ -156,7 +173,8 @@ public class BeelineHiveClient implements IHiveClient {
if ("".equals(resultSet.getString(1).trim())) {
break;
}
- partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), resultSet.getString(2).trim(), resultSet.getString(3).trim()));
+ partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(),
+ resultSet.getString(2).trim(), resultSet.getString(3).trim()));
}
builder.setPartitionColumns(partitionColumns);
}
@@ -213,7 +231,8 @@ public class BeelineHiveClient implements IHiveClient {
public static void main(String[] args) throws SQLException {
- BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://sandbox:10000'");
+ BeelineHiveClient loader = new BeelineHiveClient(
+ "-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://sandbox:10000'");
//BeelineHiveClient loader = new BeelineHiveClient(StringUtils.join(args, " "));
HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", "test_kylin_fact_part");
System.out.println(hiveTableMeta);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1d825a1e/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineHIveClientTest.java
----------------------------------------------------------------------
diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineHIveClientTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineHIveClientTest.java
new file mode 100644
index 0000000..00a25e5
--- /dev/null
+++ b/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineHIveClientTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class BeelineHIveClientTest {
+ @Test
+ public void testBasics() {
+ String dataType = "varchar";
+ String precision = "60";
+ String scale = null;
+ dataType = BeelineHiveClient.considerDataTypePrecision(dataType, precision, scale);
+ Assert.assertEquals("varchar(60)", dataType);
+
+ dataType = "char";
+ precision = "50";
+ scale = null;
+ dataType = BeelineHiveClient.considerDataTypePrecision(dataType, precision, scale);
+ Assert.assertEquals("char(50)", dataType);
+
+ dataType = "decimal";
+ precision = "8";
+ scale = "4";
+ dataType = BeelineHiveClient.considerDataTypePrecision(dataType, precision, scale);
+ Assert.assertEquals("decimal(8,4)", dataType);
+
+ dataType = "numeric";
+ precision = "7";
+ scale = "3";
+ dataType = BeelineHiveClient.considerDataTypePrecision(dataType, precision, scale);
+ Assert.assertEquals("numeric(7,3)", dataType);
+ }
+}