You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2024/05/03 14:57:31 UTC
(hudi) 37/40: [HUDI-7702] Remove unused method in ReflectUtil (#11135)
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch release-0.14.1-spark35-scala213
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit af38c325f6495a468542dd8627f06cf094f7a480
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Wed May 1 22:21:00 2024 -0700
[HUDI-7702] Remove unused method in ReflectUtil (#11135)
---
.../apache/hudi/spark3/internal/ReflectUtil.java | 29 +-----------
.../hudi/spark3/internal/TestReflectUtil.java | 54 ----------------------
.../hudi/spark3/internal/TestReflectUtil.java | 54 ----------------------
.../hudi/spark3/internal/TestReflectUtil.java | 54 ----------------------
4 files changed, 1 insertion(+), 190 deletions(-)
diff --git a/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/ReflectUtil.java b/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/ReflectUtil.java
index ad83720b021..c726777876f 100644
--- a/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/ReflectUtil.java
+++ b/hudi-spark-datasource/hudi-spark3-common/src/main/java/org/apache/hudi/spark3/internal/ReflectUtil.java
@@ -18,41 +18,14 @@
package org.apache.hudi.spark3.internal;
import org.apache.hudi.HoodieSparkUtils;
-import org.apache.spark.sql.catalyst.plans.logical.InsertIntoStatement;
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan;
-import org.apache.spark.sql.catalyst.util.DateFormatter;
-import scala.Option;
-import scala.collection.Seq;
-import scala.collection.immutable.Map;
+import org.apache.spark.sql.catalyst.util.DateFormatter;
-import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
import java.time.ZoneId;
public class ReflectUtil {
- public static InsertIntoStatement createInsertInto(LogicalPlan table, Map<String, Option<String>> partition, Seq<String> userSpecifiedCols,
- LogicalPlan query, boolean overwrite, boolean ifPartitionNotExists, boolean byName) {
- try {
- if (HoodieSparkUtils.gteqSpark3_5()) {
- Constructor<InsertIntoStatement> constructor = InsertIntoStatement.class.getConstructor(
- LogicalPlan.class, Map.class, Seq.class, LogicalPlan.class, boolean.class, boolean.class, boolean.class);
- return constructor.newInstance(table, partition, userSpecifiedCols, query, overwrite, ifPartitionNotExists, byName);
- } else if (HoodieSparkUtils.isSpark3_0()) {
- Constructor<InsertIntoStatement> constructor = InsertIntoStatement.class.getConstructor(
- LogicalPlan.class, Map.class, LogicalPlan.class, boolean.class, boolean.class);
- return constructor.newInstance(table, partition, query, overwrite, ifPartitionNotExists);
- } else {
- Constructor<InsertIntoStatement> constructor = InsertIntoStatement.class.getConstructor(
- LogicalPlan.class, Map.class, Seq.class, LogicalPlan.class, boolean.class, boolean.class);
- return constructor.newInstance(table, partition, userSpecifiedCols, query, overwrite, ifPartitionNotExists);
- }
- } catch (Exception e) {
- throw new RuntimeException("Error in create InsertIntoStatement", e);
- }
- }
-
public static DateFormatter getDateFormatter(ZoneId zoneId) {
try {
ClassLoader loader = Thread.currentThread().getContextClassLoader();
diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java b/hudi-spark-datasource/hudi-spark3.3.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java
deleted file mode 100644
index 0763a22f032..00000000000
--- a/hudi-spark-datasource/hudi-spark3.3.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi.spark3.internal;
-
-import org.apache.hudi.testutils.HoodieClientTestBase;
-
-import org.apache.spark.sql.SparkSession;
-import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation;
-import org.apache.spark.sql.catalyst.plans.logical.InsertIntoStatement;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-/**
- * Unit tests {@link ReflectUtil}.
- */
-public class TestReflectUtil extends HoodieClientTestBase {
-
- @Test
- public void testDataSourceWriterExtraCommitMetadata() throws Exception {
- SparkSession spark = sqlContext.sparkSession();
-
- String insertIntoSql = "insert into test_reflect_util values (1, 'z3', 1, '2021')";
- InsertIntoStatement statement = (InsertIntoStatement) spark.sessionState().sqlParser().parsePlan(insertIntoSql);
-
- InsertIntoStatement newStatment = ReflectUtil.createInsertInto(
- statement.table(),
- statement.partitionSpec(),
- scala.collection.immutable.List.empty(),
- statement.query(),
- statement.overwrite(),
- statement.ifPartitionNotExists(),
- false);
-
- Assertions.assertTrue(
- ((UnresolvedRelation)newStatment.table()).multipartIdentifier().contains("test_reflect_util"));
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java b/hudi-spark-datasource/hudi-spark3.4.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java
deleted file mode 100644
index 0763a22f032..00000000000
--- a/hudi-spark-datasource/hudi-spark3.4.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi.spark3.internal;
-
-import org.apache.hudi.testutils.HoodieClientTestBase;
-
-import org.apache.spark.sql.SparkSession;
-import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation;
-import org.apache.spark.sql.catalyst.plans.logical.InsertIntoStatement;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-/**
- * Unit tests {@link ReflectUtil}.
- */
-public class TestReflectUtil extends HoodieClientTestBase {
-
- @Test
- public void testDataSourceWriterExtraCommitMetadata() throws Exception {
- SparkSession spark = sqlContext.sparkSession();
-
- String insertIntoSql = "insert into test_reflect_util values (1, 'z3', 1, '2021')";
- InsertIntoStatement statement = (InsertIntoStatement) spark.sessionState().sqlParser().parsePlan(insertIntoSql);
-
- InsertIntoStatement newStatment = ReflectUtil.createInsertInto(
- statement.table(),
- statement.partitionSpec(),
- scala.collection.immutable.List.empty(),
- statement.query(),
- statement.overwrite(),
- statement.ifPartitionNotExists(),
- false);
-
- Assertions.assertTrue(
- ((UnresolvedRelation)newStatment.table()).multipartIdentifier().contains("test_reflect_util"));
- }
-}
diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java b/hudi-spark-datasource/hudi-spark3.5.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java
deleted file mode 100644
index 5a08e54f5e1..00000000000
--- a/hudi-spark-datasource/hudi-spark3.5.x/src/test/java/org/apache/hudi/spark3/internal/TestReflectUtil.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi.spark3.internal;
-
-import org.apache.hudi.testutils.HoodieClientTestBase;
-
-import org.apache.spark.sql.SparkSession;
-import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation;
-import org.apache.spark.sql.catalyst.plans.logical.InsertIntoStatement;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-/**
- * Unit tests {@link ReflectUtil}.
- */
-public class TestReflectUtil extends HoodieClientTestBase {
-
- @Test
- public void testDataSourceWriterExtraCommitMetadata() throws Exception {
- SparkSession spark = sqlContext.sparkSession();
-
- String insertIntoSql = "insert into test_reflect_util values (1, 'z3', 1, '2021')";
- InsertIntoStatement statement = (InsertIntoStatement) spark.sessionState().sqlParser().parsePlan(insertIntoSql);
-
- InsertIntoStatement newStatment = ReflectUtil.createInsertInto(
- statement.table(),
- statement.partitionSpec(),
- scala.collection.immutable.List.empty(),
- statement.query(),
- statement.overwrite(),
- statement.ifPartitionNotExists(),
- statement.byName());
-
- Assertions.assertTrue(
- ((UnresolvedRelation)newStatment.table()).multipartIdentifier().contains("test_reflect_util"));
- }
-}