You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by ja...@apache.org on 2023/03/21 11:45:04 UTC
[linkis] branch dev-1.3.2 updated: Backport hive compatibility to 1.3.2 (#4394)
This is an automated email from the ASF dual-hosted git repository.
jackxu2011 pushed a commit to branch dev-1.3.2
in repository https://gitbox.apache.org/repos/asf/linkis.git
The following commit(s) were added to refs/heads/dev-1.3.2 by this push:
new bfee7f1c0 Backport hive compatibility to 1.3.2 (#4394)
bfee7f1c0 is described below
commit bfee7f1c0dbf6e61b98d767278aea44dad3d7677
Author: GuoPhilipse <46...@users.noreply.github.com>
AuthorDate: Tue Mar 21 19:44:58 2023 +0800
Backport hive compatibility to 1.3.2 (#4394)
* backport hive compatibility #4255 #4246 #4341 to 1.3.2
* trigger rebuild
---
.../label/entity/engine/EngineTypeLabel.java | 15 +++++++
.../label/entity/engine/EngineTypeLabelTest.java | 47 ++++++++++++++++++++
linkis-engineconn-plugins/flink/pom.xml | 1 -
linkis-engineconn-plugins/hive/pom.xml | 4 --
.../hive/serde/CustomerDelimitedJSONSerDe.java | 50 +++++++++++++++++++++-
.../linkis-metadata-query/service/hive/pom.xml | 1 -
pom.xml | 1 +
7 files changed, 112 insertions(+), 7 deletions(-)
diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java
index 09492b146..912f6c9f9 100644
--- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java
+++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java
@@ -74,4 +74,19 @@ public class EngineTypeLabel extends GenericLabel implements EngineNodeLabel, EM
public Boolean isEmpty() {
return StringUtils.isBlank(getEngineType()) || StringUtils.isBlank(getVersion());
}
+
+ @Override
+ protected void setStringValue(String stringValue) {
+ String version;
+ String engineType = stringValue.split("-")[0];
+
+ if (engineType.equals("*")) {
+ version = stringValue.replaceFirst("[" + engineType + "]-", "");
+ } else {
+ version = stringValue.replaceFirst(engineType + "-", "");
+ }
+
+ setEngineType(engineType);
+ setVersion(version);
+ }
}
diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabelTest.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabelTest.java
new file mode 100644
index 000000000..cc4171b43
--- /dev/null
+++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabelTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.manager.label.entity.engine;
+
+import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactory;
+import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/** EngineTypeLabel Tester */
+public class EngineTypeLabelTest {
+
+ @Test
+ public void testSetStringValue() {
+ String engineType = "hive";
+ String version = "1.1.0-cdh5.12.0";
+
+ String engineType1 = "*";
+ String version1 = "*";
+
+ LabelBuilderFactory labelBuilderFactory = LabelBuilderFactoryContext.getLabelBuilderFactory();
+ EngineTypeLabel engineTypeLabel = labelBuilderFactory.createLabel(EngineTypeLabel.class);
+ engineTypeLabel.setStringValue(engineType + "-" + version);
+ Assertions.assertEquals(engineTypeLabel.getEngineType(), engineType);
+ Assertions.assertEquals(engineTypeLabel.getVersion(), version);
+
+ engineTypeLabel.setStringValue(engineType1 + "-" + version1);
+ Assertions.assertEquals(engineTypeLabel.getEngineType(), engineType1);
+ Assertions.assertEquals(engineTypeLabel.getVersion(), version1);
+ }
+}
diff --git a/linkis-engineconn-plugins/flink/pom.xml b/linkis-engineconn-plugins/flink/pom.xml
index 3ceca971e..2d88c95b6 100644
--- a/linkis-engineconn-plugins/flink/pom.xml
+++ b/linkis-engineconn-plugins/flink/pom.xml
@@ -27,7 +27,6 @@
<artifactId>linkis-engineconn-plugin-flink</artifactId>
<properties>
<flink.version>1.12.2</flink.version>
- <hive.version>2.3.3</hive.version>
<commons-cli.version>1.3.1</commons-cli.version>
</properties>
diff --git a/linkis-engineconn-plugins/hive/pom.xml b/linkis-engineconn-plugins/hive/pom.xml
index 8fe446167..28b60fff0 100644
--- a/linkis-engineconn-plugins/hive/pom.xml
+++ b/linkis-engineconn-plugins/hive/pom.xml
@@ -26,10 +26,6 @@
<artifactId>linkis-engineplugin-hive</artifactId>
- <properties>
- <hive.version>2.3.3</hive.version>
- </properties>
-
<dependencies>
<dependency>
diff --git a/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java b/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java
index 671b0c1d1..897a76e66 100644
--- a/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java
+++ b/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java
@@ -17,6 +17,8 @@
package org.apache.linkis.engineplugin.hive.serde;
+import org.apache.linkis.common.utils.ClassUtils;
+
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.hive.serde2.ByteStream;
import org.apache.hadoop.hive.serde2.SerDeException;
@@ -33,6 +35,7 @@ import org.apache.hadoop.io.WritableComparable;
import java.io.IOException;
import java.io.OutputStream;
+import java.lang.reflect.InvocationTargetException;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -329,7 +332,52 @@ public class CustomerDelimitedJSONSerDe extends LazySimpleSerDe {
}
default:
{
- throw new RuntimeException("Unknown primitive type: " + category);
+ if (!"INTERVAL_YEAR_MONTH".equals(category.name())
+ && !"INTERVAL_DAY_TIME".equals(category.name())) {
+ throw new RuntimeException("Unknown primitive type: " + category);
+ }
+ boolean containsIntervalYearMonth = false;
+ boolean containsIntervalDayTime = false;
+ for (PrimitiveObjectInspector.PrimitiveCategory primitiveCategory :
+ PrimitiveObjectInspector.PrimitiveCategory.values()) {
+ containsIntervalYearMonth =
+ "INTERVAL_YEAR_MONTH".equals(primitiveCategory.name())
+ && "INTERVAL_YEAR_MONTH".equals(category.name());
+ containsIntervalDayTime =
+ "INTERVAL_DAY_TIME".equals(primitiveCategory.name())
+ && "INTERVAL_DAY_TIME".equals(category.name());
+ try {
+ if (containsIntervalYearMonth) {
+ wc =
+ (WritableComparable)
+ ClassUtils.getClassInstance(
+ "org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalYearMonthObjectInspector")
+ .getClass()
+ .getMethod("getPrimitiveWritableObject", Object.class)
+ .invoke(oi, o);
+ binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes());
+ break;
+ }
+ if (containsIntervalDayTime) {
+ wc =
+ (WritableComparable)
+ ClassUtils.getClassInstance(
+ "org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalDayTimeObjectInspector")
+ .getClass()
+ .getMethod("getPrimitiveWritableObject", Object.class)
+ .invoke(oi, o);
+ binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes());
+ break;
+ }
+ } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
+ LOG.error("Fail to invoke method:[getPrimitiveWritableObject]!", e);
+ }
+ }
+ if (containsIntervalYearMonth || containsIntervalDayTime) {
+ break;
+ } else {
+ throw new RuntimeException("Unknown primitive type: " + category);
+ }
}
}
if (binaryData == null) {
diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml
index 127913944..b0c9cdef3 100644
--- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml
+++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml
@@ -26,7 +26,6 @@
<artifactId>linkis-metadata-query-service-hive</artifactId>
<properties>
- <hive.version>2.3.3</hive.version>
<hadoop.version>2.7.2</hadoop.version>
<datanucleus-api-jdo.version>4.2.4</datanucleus-api-jdo.version>
</properties>
diff --git a/pom.xml b/pom.xml
index 5b9cb09c9..4f2381c87 100644
--- a/pom.xml
+++ b/pom.xml
@@ -106,6 +106,7 @@
<revision>1.3.2-SNAPSHOT</revision>
<jedis.version>2.9.2</jedis.version>
<spark.version>2.4.3</spark.version>
+ <hive.version>2.3.3</hive.version>
<hadoop.version>2.7.2</hadoop.version>
<hadoop-hdfs-client.artifact>hadoop-hdfs</hadoop-hdfs-client.artifact>
<hadoop-hdfs-client-shade.version>2.7.2</hadoop-hdfs-client-shade.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org