You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by xu...@apache.org on 2023/06/29 12:35:27 UTC
[hudi] branch master updated: [MINOR] Improve CollectionUtils helper methods (#9088)
This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 8def3e68ae5 [MINOR] Improve CollectionUtils helper methods (#9088)
8def3e68ae5 is described below
commit 8def3e68ae5a0b72eefe26db49b6d33226f7b4c0
Author: Shiyan Xu <27...@users.noreply.github.com>
AuthorDate: Thu Jun 29 05:35:19 2023 -0700
[MINOR] Improve CollectionUtils helper methods (#9088)
---
.../action/clean/CleanPlanActionExecutor.java | 4 +--
.../action/commit/TestSchemaEvolutionClient.java | 3 +-
.../table/action/rollback/TestRollbackUtils.java | 3 +-
.../table/functional/TestCleanPlanExecutor.java | 2 +-
.../apache/hudi/common/util/CollectionUtils.java | 35 +++++++---------------
.../hudi/common/table/TestTimelineUtils.java | 2 +-
.../table/view/TestIncrementalFSViewSync.java | 3 +-
.../hudi/common/testutils/HoodieTestTable.java | 8 ++---
8 files changed, 23 insertions(+), 37 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanActionExecutor.java
index 043db1acbf9..ba7c71b1356 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanActionExecutor.java
@@ -29,7 +29,6 @@ import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
import org.apache.hudi.common.util.CleanerUtils;
-import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
@@ -42,6 +41,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -132,7 +132,7 @@ public class CleanPlanActionExecutor<T, I, K, O> extends BaseActionExecutor<T, I
return new HoodieCleanerPlan(earliestInstant
.map(x -> new HoodieActionInstant(x.getTimestamp(), x.getAction(), x.getState().name())).orElse(null),
planner.getLastCompletedCommitTimestamp(),
- config.getCleanerPolicy().name(), CollectionUtils.createImmutableMap(),
+ config.getCleanerPolicy().name(), Collections.emptyMap(),
CleanPlanner.LATEST_CLEAN_PLAN_VERSION, cleanOps, partitionsToDelete);
} catch (IOException e) {
throw new HoodieIOException("Failed to schedule clean operation", e);
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestSchemaEvolutionClient.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestSchemaEvolutionClient.java
index bf825df570f..dc45a80754b 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestSchemaEvolutionClient.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/table/action/commit/TestSchemaEvolutionClient.java
@@ -24,7 +24,6 @@ import org.apache.hudi.common.model.HoodieAvroRecord;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.table.TableSchemaResolver;
import org.apache.hudi.common.testutils.RawTripTestPayload;
-import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.internal.schema.Types;
import org.apache.hudi.testutils.HoodieJavaClientTestHarness;
@@ -72,7 +71,7 @@ public class TestSchemaEvolutionClient extends HoodieJavaClientTestHarness {
.withEngineType(EngineType.JAVA)
.withPath(basePath)
.withSchema(SCHEMA.toString())
- .withProps(CollectionUtils.createImmutableMap(HoodieWriteConfig.TBL_NAME.key(), "hoodie_test_table"))
+ .withProps(Collections.singletonMap(HoodieWriteConfig.TBL_NAME.key(), "hoodie_test_table"))
.build();
return new HoodieJavaWriteClient<>(context, config);
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackUtils.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackUtils.java
index f03d9f3967d..c22a2aef424 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackUtils.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackUtils.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.junit.jupiter.api.Test;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.stream.Collectors;
@@ -117,7 +118,7 @@ public class TestRollbackUtils {
assertIterableEquals(CollectionUtils.createImmutableList(partitionPath1 + "dataFile1" + BASE_FILE_EXTENSION,
partitionPath1 + "dataFile2" + BASE_FILE_EXTENSION, partitionPath1 + "dataFile1.log").stream().sorted().collect(Collectors.toList()),
dataFilesOnlyStatMerge2.getSuccessDeleteFiles().stream().sorted().collect(Collectors.toList()));
- assertEquals(CollectionUtils.createImmutableMap(generateFileStatus(partitionPath1 + "dataFile1.log"), 10L),
+ assertEquals(Collections.singletonMap(generateFileStatus(partitionPath1 + "dataFile1.log"), 10L),
dataFilesOnlyStatMerge2.getCommandBlocksCount());
}
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestCleanPlanExecutor.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestCleanPlanExecutor.java
index 9d9d27e1970..bbc57367231 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestCleanPlanExecutor.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestCleanPlanExecutor.java
@@ -235,7 +235,7 @@ public class TestCleanPlanExecutor extends HoodieCleanerTestBase {
// No cleaning on partially written file, with no commit.
testTable.forCommit("00000000000011").withBaseFilesInPartition(p0, file3P0C2);
- HoodieCommitMetadata commitMetadata = generateCommitMetadata("00000000000011", CollectionUtils.createImmutableMap(p0,
+ HoodieCommitMetadata commitMetadata = generateCommitMetadata("00000000000011", Collections.singletonMap(p0,
CollectionUtils.createImmutableList(file3P0C2)));
metaClient.getActiveTimeline().createNewInstant(
new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.COMMIT_ACTION, "00000000000011"));
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/CollectionUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/CollectionUtils.java
index 59e25e8afbe..3ec7040e95a 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/CollectionUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/CollectionUtils.java
@@ -243,40 +243,31 @@ public class CollectionUtils {
return !iterator2.hasNext();
}
- @SafeVarargs
- public static <T> Set<T> createSet(final T... elements) {
- return Stream.of(elements).collect(Collectors.toSet());
- }
-
- public static <K,V> Map<K, V> createImmutableMap(final K key, final V value) {
- return Collections.unmodifiableMap(Collections.singletonMap(key, value));
- }
-
@SafeVarargs
public static <T> List<T> createImmutableList(final T... elements) {
return Collections.unmodifiableList(Stream.of(elements).collect(Collectors.toList()));
}
- public static <K,V> Map<K,V> createImmutableMap(final Map<K,V> map) {
- return Collections.unmodifiableMap(map);
+ public static <T> List<T> createImmutableList(final List<T> list) {
+ return Collections.unmodifiableList(list);
}
@SafeVarargs
- public static <K,V> Map<K,V> createImmutableMap(final Pair<K,V>... elements) {
- Map<K,V> map = new HashMap<>();
- for (Pair<K,V> pair: elements) {
+ public static <K, V> Map<K, V> createImmutableMap(final Pair<K, V>... elements) {
+ Map<K, V> map = new HashMap<>();
+ for (Pair<K, V> pair : elements) {
map.put(pair.getLeft(), pair.getRight());
}
return Collections.unmodifiableMap(map);
}
+ public static <K, V> Map<K, V> createImmutableMap(final Map<K, V> map) {
+ return Collections.unmodifiableMap(map);
+ }
+
@SafeVarargs
- public static <K,V> HashMap<K, V> createHashMap(final Pair<K, V>... elements) {
- HashMap<K,V> map = new HashMap<>();
- for (Pair<K,V> pair: elements) {
- map.put(pair.getLeft(), pair.getRight());
- }
- return map;
+ public static <T> Set<T> createSet(final T... elements) {
+ return Stream.of(elements).collect(Collectors.toSet());
}
@SafeVarargs
@@ -288,10 +279,6 @@ public class CollectionUtils {
return Collections.unmodifiableSet(set);
}
- public static <T> List<T> createImmutableList(final List<T> list) {
- return Collections.unmodifiableList(list);
- }
-
private static Object[] checkElementsNotNull(Object... array) {
return checkElementsNotNull(array, array.length);
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java
index a1f00b8eaf0..3c4f842d376 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java
@@ -491,7 +491,7 @@ public class TestTimelineUtils extends HoodieCommonTestHarness {
List<HoodieInstant> rollbackInstants = new ArrayList<>();
rollbackInstants.add(new HoodieInstant(false, commitTs, actionType));
HoodieRestoreMetadata metadata = TimelineMetadataUtils.convertRestoreMetadata(commitTs, 200, rollbackInstants,
- CollectionUtils.createImmutableMap(commitTs, rollbackM));
+ Collections.singletonMap(commitTs, rollbackM));
return TimelineMetadataUtils.serializeRestoreMetadata(metadata).get();
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
index 560f209ea03..f04f4505846 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
@@ -44,7 +44,6 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.common.util.CleanerUtils;
-import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.CompactionUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
@@ -655,7 +654,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
List<HoodieRollbackMetadata> rollbackM = new ArrayList<>();
rollbackM.add(rollbackMetadata);
HoodieRestoreMetadata metadata = TimelineMetadataUtils.convertRestoreMetadata(rollbackInstant,
- 100, Collections.singletonList(instant), CollectionUtils.createImmutableMap(rollbackInstant, rollbackM));
+ 100, Collections.singletonList(instant), Collections.singletonMap(rollbackInstant, rollbackM));
HoodieInstant restoreInstant = new HoodieInstant(true, HoodieTimeline.RESTORE_ACTION, rollbackInstant);
metaClient.getActiveTimeline().createNewInstant(restoreInstant);
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
index 63c4e15ce2b..a6775889970 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
@@ -87,6 +87,7 @@ import java.util.stream.IntStream;
import java.util.stream.Stream;
import static java.time.temporal.ChronoUnit.SECONDS;
+import static java.util.Collections.singletonMap;
import static org.apache.hudi.common.config.HoodieMetadataConfig.DEFAULT_METADATA_POPULATE_META_FIELDS;
import static org.apache.hudi.common.model.HoodieTableType.MERGE_ON_READ;
import static org.apache.hudi.common.model.WriteOperationType.CLUSTER;
@@ -121,7 +122,6 @@ import static org.apache.hudi.common.testutils.FileCreateUtils.createSavepointCo
import static org.apache.hudi.common.testutils.FileCreateUtils.deleteSavepointCommit;
import static org.apache.hudi.common.testutils.FileCreateUtils.logFileName;
import static org.apache.hudi.common.util.CleanerUtils.convertCleanMetadata;
-import static org.apache.hudi.common.util.CollectionUtils.createImmutableMap;
import static org.apache.hudi.common.util.CommitUtils.buildMetadata;
import static org.apache.hudi.common.util.CommitUtils.getCommitActionType;
import static org.apache.hudi.common.util.StringUtils.EMPTY_STRING;
@@ -263,7 +263,7 @@ public class HoodieTestTable {
if (MERGE_ON_READ.equals(metaClient.getTableType()) && UPSERT.equals(operationType)) {
writeStats.addAll(generateHoodieWriteStatForPartitionLogFiles(testTableState.getPartitionToLogFileInfoMap(commitTime), commitTime, bootstrap));
}
- Map<String, String> extraMetadata = createImmutableMap("test", "test");
+ Map<String, String> extraMetadata = singletonMap("test", "test");
return buildMetadata(writeStats, partitionToReplaceFileIds, Option.of(extraMetadata), operationType, PHONY_TABLE_SCHEMA, action);
}
@@ -414,7 +414,7 @@ public class HoodieTestTable {
String fileId = UUID.randomUUID().toString();
String logFileName = logFileName(instantTimeToDelete, fileId, 0);
FileCreateUtils.createLogFile(basePath, entry.getKey(), instantTimeToDelete, fileId, 0, (int) rollbackLogFileSize);
- rollbackPartitionMetadata.setRollbackLogFiles(createImmutableMap(logFileName, rollbackLogFileSize));
+ rollbackPartitionMetadata.setRollbackLogFiles(singletonMap(logFileName, rollbackLogFileSize));
partitionMetadataMap.put(entry.getKey(), rollbackPartitionMetadata);
}
rollbackMetadata.setPartitionMetadata(partitionMetadataMap);
@@ -1023,7 +1023,7 @@ public class HoodieTestTable {
Map<String, List<Pair<String, Integer>>> partitionToFilesNameLengthMap,
boolean bootstrap, boolean createInflightCommit) throws Exception {
if (partitionToFilesNameLengthMap.isEmpty()) {
- partitionToFilesNameLengthMap = Collections.singletonMap(EMPTY_STRING, Collections.EMPTY_LIST);
+ partitionToFilesNameLengthMap = singletonMap(EMPTY_STRING, Collections.emptyList());
}
HoodieTestTableState testTableState = getTestTableStateWithPartitionFileInfo(operationType,
metaClient.getTableType(), commitTime, partitionToFilesNameLengthMap);