You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2024/11/28 06:00:57 UTC
(hadoop) 33/41: Integration of TOS: Fix checkstyle.
This is an automated email from the ASF dual-hosted git repository.
jinglun pushed a commit to branch HADOOP-19236
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 0c0981ef870393b498fa9b3cd7f088b4bee5ef28
Author: lijinglun <li...@bytedance.com>
AuthorDate: Mon Nov 11 11:30:13 2024 +0800
Integration of TOS: Fix checkstyle.
---
.../org/apache/hadoop/fs/tosfs/RawFSUtils.java | 2 +-
.../org/apache/hadoop/fs/tosfs/RawFileSystem.java | 6 +-
.../hadoop/fs/tosfs/commit/CommitContext.java | 2 +-
.../apache/hadoop/fs/tosfs/commit/CommitUtils.java | 4 +-
.../apache/hadoop/fs/tosfs/commit/Committer.java | 4 +-
.../hadoop/fs/tosfs/commit/MagicOutputStream.java | 4 +-
.../org/apache/hadoop/fs/tosfs/commit/Pending.java | 4 +-
.../apache/hadoop/fs/tosfs/commit/PendingSet.java | 2 +-
.../apache/hadoop/fs/tosfs/commit/SuccessData.java | 2 +-
.../hadoop/fs/tosfs/commit/mapred/Committer.java | 2 +-
.../org/apache/hadoop/fs/tosfs/conf/ConfKeys.java | 2 +-
.../tosfs/object/ObjectMultiRangeInputStream.java | 2 +-
.../hadoop/fs/tosfs/object/ObjectOutputStream.java | 4 +-
.../fs/tosfs/object/ObjectRangeInputStream.java | 2 +-
.../fs/tosfs/object/staging/FileStagingPart.java | 2 +-
.../apache/hadoop/fs/tosfs/ops/DefaultFsOps.java | 6 +-
.../apache/hadoop/fs/tosfs/ops/DirectoryFsOps.java | 7 +-
.../org/apache/hadoop/fs/tosfs/ops/RenameOp.java | 4 +-
.../org/apache/hadoop/fs/tosfs/util/FSUtils.java | 2 +-
.../org/apache/hadoop/fs/tosfs/util/Iterables.java | 126 ++++++++++++++++
.../fs/tosfs/object/TestObjectOutputStream.java | 2 +-
.../hadoop/fs/tosfs/object/TestObjectStorage.java | 4 +-
.../hadoop/fs/tosfs/ops/TestDefaultFsOps.java | 2 +-
.../org/apache/hadoop/fs/tosfs/util/TempFiles.java | 2 +-
.../apache/hadoop/fs/tosfs/util/TestIterables.java | 165 +++++++++++++++++++++
25 files changed, 328 insertions(+), 36 deletions(-)
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/RawFSUtils.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/RawFSUtils.java
index a80c03dfcc4..e8cd9c7b793 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/RawFSUtils.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/RawFSUtils.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.fs.tosfs;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
import java.util.Objects;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/RawFileSystem.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/RawFileSystem.java
index b89a1477ee6..fe90bcba234 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/RawFileSystem.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/RawFileSystem.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.fs.tosfs;
-import com.google.common.collect.Iterators;
import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -61,9 +60,10 @@ import org.apache.hadoop.fs.tosfs.util.FuseUtils;
import org.apache.hadoop.fs.tosfs.util.Range;
import org.apache.hadoop.fs.tosfs.util.RemoteIterators;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Iterators;
import org.apache.hadoop.util.DataChecksum;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.util.Progressable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitContext.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitContext.java
index 1dc67bc27c3..40d371bcf88 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitContext.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitContext.java
@@ -17,7 +17,7 @@
package org.apache.hadoop.fs.tosfs.commit;
import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
import java.util.List;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitUtils.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitUtils.java
index 845116344db..f3a9f6c301e 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitUtils.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitUtils.java
@@ -31,8 +31,8 @@ import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/Committer.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/Committer.java
index 6180447ed34..6d016609762 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/Committer.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/Committer.java
@@ -36,8 +36,8 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.thirdparty.com.google.common.base.MoreObjects;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/MagicOutputStream.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/MagicOutputStream.java
index d78d9466079..eda291c1b59 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/MagicOutputStream.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/MagicOutputStream.java
@@ -25,8 +25,8 @@ import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
import org.apache.hadoop.fs.tosfs.object.ObjectStorageFactory;
import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
import org.apache.hadoop.fs.tosfs.object.Part;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
import java.io.IOException;
import java.util.Arrays;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/Pending.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/Pending.java
index 5218912ed66..0c595dabe65 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/Pending.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/Pending.java
@@ -21,8 +21,8 @@ import org.apache.hadoop.fs.tosfs.object.Part;
import org.apache.hadoop.fs.tosfs.util.JsonCodec;
import org.apache.hadoop.fs.tosfs.util.Serializer;
import org.apache.hadoop.thirdparty.com.google.common.base.MoreObjects;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
import java.io.IOException;
import java.util.List;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/PendingSet.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/PendingSet.java
index a6ca93d4d3e..c68db9718ac 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/PendingSet.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/PendingSet.java
@@ -21,8 +21,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.tosfs.util.JsonCodec;
import org.apache.hadoop.fs.tosfs.util.Serializer;
import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
+import org.apache.hadoop.util.Lists;
import java.io.IOException;
import java.io.UncheckedIOException;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/SuccessData.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/SuccessData.java
index 199d084f5a7..9c1e376a039 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/SuccessData.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/SuccessData.java
@@ -21,7 +21,7 @@ import org.apache.hadoop.fs.tosfs.util.Serializer;
import org.apache.hadoop.thirdparty.com.google.common.base.MoreObjects;
import org.apache.hadoop.thirdparty.com.google.common.base.Throwables;
import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
import org.apache.hadoop.util.StringUtils;
import java.io.IOException;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/mapred/Committer.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/mapred/Committer.java
index 614ee70ac34..5a512d71acc 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/mapred/Committer.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/mapred/Committer.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobContext;
import org.apache.hadoop.mapred.JobStatus;
import org.apache.hadoop.mapred.TaskAttemptContext;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/conf/ConfKeys.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/conf/ConfKeys.java
index 517f308c91f..990033c361a 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/conf/ConfKeys.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/conf/ConfKeys.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.tosfs.conf;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
public class ConfKeys {
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectMultiRangeInputStream.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectMultiRangeInputStream.java
index 99ed5bf23a7..4a9357e46bf 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectMultiRangeInputStream.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectMultiRangeInputStream.java
@@ -25,8 +25,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.tosfs.util.CommonUtils;
import org.apache.hadoop.fs.tosfs.util.FSUtils;
import org.apache.hadoop.fs.tosfs.util.Range;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints;
+import org.apache.hadoop.util.Preconditions;
import java.io.EOFException;
import java.io.IOException;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectOutputStream.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectOutputStream.java
index d6fbdebe8f4..aa2bffb6998 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectOutputStream.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectOutputStream.java
@@ -23,8 +23,8 @@ import org.apache.hadoop.fs.tosfs.object.staging.FileStagingPart;
import org.apache.hadoop.fs.tosfs.object.staging.StagingPart;
import org.apache.hadoop.fs.tosfs.util.CommonUtils;
import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectRangeInputStream.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectRangeInputStream.java
index d33d0079e4f..77d180323ff 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectRangeInputStream.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectRangeInputStream.java
@@ -23,9 +23,9 @@ import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.tosfs.util.FSUtils;
import org.apache.hadoop.fs.tosfs.util.Range;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
import org.apache.hadoop.thirdparty.com.google.common.io.ByteStreams;
import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints;
+import org.apache.hadoop.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/staging/FileStagingPart.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/staging/FileStagingPart.java
index 7f70c0820e3..3dda0aab904 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/staging/FileStagingPart.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/staging/FileStagingPart.java
@@ -18,7 +18,7 @@ package org.apache.hadoop.fs.tosfs.object.staging;
import org.apache.hadoop.fs.tosfs.util.CommonUtils;
import org.apache.hadoop.thirdparty.com.google.common.base.MoreObjects;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/ops/DefaultFsOps.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/ops/DefaultFsOps.java
index 09541f9fe6e..55eedeec836 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/ops/DefaultFsOps.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/ops/DefaultFsOps.java
@@ -30,15 +30,15 @@ import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
import org.apache.hadoop.fs.tosfs.object.request.ListObjectsRequest;
import org.apache.hadoop.fs.tosfs.object.response.ListObjectsResponse;
import org.apache.hadoop.fs.tosfs.util.CommonUtils;
-import org.apache.hadoop.thirdparty.com.google.common.base.Function;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.fs.tosfs.util.Iterables;
+import org.apache.hadoop.util.Lists;
import java.io.IOException;
import java.util.Date;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
+import java.util.function.Function;
import java.util.function.Predicate;
import static org.apache.hadoop.fs.tosfs.object.ObjectUtils.SLASH;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/ops/DirectoryFsOps.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/ops/DirectoryFsOps.java
index 3f1d829b03e..5b2958378ba 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/ops/DirectoryFsOps.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/ops/DirectoryFsOps.java
@@ -24,10 +24,10 @@ import org.apache.hadoop.fs.tosfs.RawFileStatus;
import org.apache.hadoop.fs.tosfs.object.DirectoryStorage;
import org.apache.hadoop.fs.tosfs.object.ObjectInfo;
import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
-import org.apache.hadoop.thirdparty.com.google.common.base.Function;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables;
+import org.apache.hadoop.fs.tosfs.util.Iterables;
import java.io.IOException;
+import java.util.function.Function;
import java.util.function.Predicate;
/**
@@ -86,7 +86,8 @@ public class DirectoryFsOps implements FsOps {
@Override
public Iterable<RawFileStatus> listDir(Path dir, boolean recursive, Predicate<String> postFilter) {
String key = ObjectUtils.pathToKey(dir, true);
- Iterable<ObjectInfo> objs = Iterables.filter(storage.listDir(key, recursive), obj -> postFilter.test(obj.key()));
+ Iterable<ObjectInfo> objs =
+ Iterables.filter(storage.listDir(key, recursive), obj -> postFilter.test(obj.key()));
return Iterables.transform(objs, objMapper);
}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/ops/RenameOp.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/ops/RenameOp.java
index f7feb85eb2d..eb0d36103e7 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/ops/RenameOp.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/ops/RenameOp.java
@@ -28,8 +28,8 @@ import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
import org.apache.hadoop.fs.tosfs.object.Part;
import org.apache.hadoop.fs.tosfs.util.CommonUtils;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/FSUtils.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/FSUtils.java
index 5c3400c953c..e521b73d6ee 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/FSUtils.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/FSUtils.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.fs.tosfs.util;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSExceptionMessages;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.util.ReflectionUtils;
import java.io.IOException;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/Iterables.java b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/Iterables.java
new file mode 100644
index 00000000000..79f0f7b6d44
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/Iterables.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.util;
+
+import org.apache.hadoop.util.Preconditions;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.function.Function;
+import java.util.function.Predicate;
+
+public class Iterables {
+
+ public static <F, T> Iterable<T> transform(final Iterable<F> fromIterable,
+ final Function<? super F, ? extends T> function) {
+ Preconditions.checkNotNull(fromIterable);
+ Preconditions.checkNotNull(function);
+ return () -> new Iterator<T>() {
+ private Iterator<F> iterator = fromIterable.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public T next() {
+ return function.apply(iterator.next());
+ }
+ };
+ }
+
+ public static <T> Iterable<T> filter(final Iterable<T> unfiltered,
+ final Predicate<? super T> predicate) {
+ Preconditions.checkNotNull(unfiltered);
+ Preconditions.checkNotNull(predicate);
+ return () -> new Iterator<T>() {
+ private Iterator<T> iterator = unfiltered.iterator();
+ private boolean advance = true;
+ private T value;
+
+ @Override
+ public boolean hasNext() {
+ if (!advance) {
+ return true;
+ }
+
+ while (iterator.hasNext()) {
+ value = iterator.next();
+ if (predicate.test(value)) {
+ advance = false;
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ @Override
+ public T next() {
+ if (hasNext()) {
+ advance = true;
+ return value;
+ }
+ throw new NoSuchElementException("No more items in iterator.");
+ }
+ };
+ }
+
+ public static <T extends @Nullable Object> Iterable<T> concat(
+ Iterable<? extends Iterable<? extends T>> inputs) {
+ return () -> new ConcatenatedIterator<>(inputs.iterator());
+ }
+
+ private static class ConcatenatedIterator<T> implements Iterator<T> {
+ // Iterators is the iterator of iterables.
+ private final Iterator<? extends Iterable<? extends T>> iterators;
+ private Iterator<? extends T> curIter;
+
+ ConcatenatedIterator(Iterator<? extends Iterable<? extends T>> iterators) {
+ Preconditions.checkNotNull(iterators, "Iterators should not be null.");
+ this.iterators = iterators;
+ }
+
+ @Override
+ public boolean hasNext() {
+ while (curIter == null || !curIter.hasNext()) {
+ if (curIter != null) {
+ curIter = null;
+ }
+
+ if (!iterators.hasNext()) {
+ return false;
+ }
+
+ curIter = iterators.next().iterator();
+ }
+ return true;
+ }
+
+ @Override
+ public T next() {
+ if (hasNext()) {
+ return curIter.next();
+ }
+ throw new NoSuchElementException("No more elements");
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectOutputStream.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectOutputStream.java
index ff1dd89d186..4fb2939d396 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectOutputStream.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectOutputStream.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.tosfs.util.FSUtils;
import org.apache.hadoop.fs.tosfs.util.TempFiles;
import org.apache.hadoop.fs.tosfs.util.TestUtility;
import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectStorage.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectStorage.java
index 4798fd2862f..35875c879d6 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectStorage.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectStorage.java
@@ -31,9 +31,9 @@ import org.apache.hadoop.fs.tosfs.util.CommonUtils;
import org.apache.hadoop.fs.tosfs.util.TempFiles;
import org.apache.hadoop.fs.tosfs.util.TestUtility;
import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestDefaultFsOps.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestDefaultFsOps.java
index 23d3cde0bef..ef5f4d3a9e5 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestDefaultFsOps.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestDefaultFsOps.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
import org.apache.hadoop.fs.tosfs.object.ObjectStorageFactory;
import org.apache.hadoop.fs.tosfs.util.TestUtility;
import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.runner.RunWith;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TempFiles.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TempFiles.java
index 81b59a65659..8d4be33014c 100644
--- a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TempFiles.java
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TempFiles.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.fs.tosfs.util;
import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
import java.io.Closeable;
import java.io.File;
diff --git a/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TestIterables.java b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TestIterables.java
new file mode 100644
index 00000000000..6b192788b2a
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TestIterables.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.util;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Function;
+import java.util.function.Predicate;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+public class TestIterables {
+
+ @Test
+ public void testTransform() {
+ List<Integer> list = Arrays.asList(1, 2, 3, 4, 5);
+ Function<Integer, Integer> transform = i -> i + 10;
+ Iterator<Integer> iter = Iterables.transform(list, transform).iterator();
+
+ for (int i = 0; i < 5; i++) {
+ assertTrue(iter.hasNext());
+ int value = iter.next();
+ assertEquals(10 + i + 1, value);
+ }
+ assertFalse(iter.hasNext());
+ }
+
+ @Test
+ public void testTransformEmptyIterable() {
+ List<Integer> list = Arrays.asList();
+ Function<Integer, Integer> transform = i -> i + 10;
+ Iterator<Integer> iter = Iterables.transform(list, transform).iterator();
+
+ assertFalse(iter.hasNext());
+ }
+
+ @Test
+ public void testFilter() {
+ // Filter odd elements.
+ List<Integer> list = Arrays.asList(1, 2, 3, 4, 5);
+ Predicate<Integer> filter = i -> (i % 2) == 0;
+ Iterator<Integer> iter = Iterables.filter(list, filter).iterator();
+
+ for (int i = 0; i < 2; i++) {
+ assertTrue(iter.hasNext());
+ int value = iter.next();
+ assertEquals((i + 1) * 2, value);
+ }
+ assertFalse(iter.hasNext());
+
+ // Ignore all elements.
+ filter = i -> false;
+ iter = Iterables.filter(list, filter).iterator();
+ assertFalse(iter.hasNext());
+ }
+
+ @Test
+ public void testFilterEmptyIterable() {
+ List<Integer> list = Arrays.asList();
+ Predicate<Integer> filter = i -> (i % 2) == 0;
+ Iterator<Integer> iter = Iterables.filter(list, filter).iterator();
+
+ assertFalse(iter.hasNext());
+ }
+
+ // Full iterators.
+ @Test
+ public void testConcatFullIterators() {
+ List<Integer> expectedList = new ArrayList<>();
+ List<Iterable<Integer>> iterList = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<Integer> list = new ArrayList<>();
+ for (int j = 0; j < 10; j++) {
+ list.add(i * 10 + j);
+ expectedList.add(i * 10 + j);
+ }
+ iterList.add(list);
+ }
+
+ verifyConcat(expectedList.iterator(), iterList);
+ }
+
+ // Empty iterators.
+ @Test
+ public void testConcatEmptyIterators() {
+ List<Integer> expectedList = new ArrayList<>();
+ List<Iterable<Integer>> iterList = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ iterList.add(Collections.emptyList());
+ }
+
+ verifyConcat(expectedList.iterator(), iterList);
+ }
+
+ // Mix full and empty iterators.
+ @Test
+ public void testConcatMixFullAndEmptyIterators() {
+ List<Integer> expectedList = new ArrayList<>();
+ List<Iterable<Integer>> iterList = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ List<Integer> list = new ArrayList<>();
+ for (int j = 0; j < 10; j++) {
+ list.add(i * 10 + j);
+ expectedList.add(i * 10 + j);
+ }
+ iterList.add(list);
+ iterList.add(Collections.emptyList());
+ iterList.add(Collections.emptyList());
+ }
+
+ verifyConcat(expectedList.iterator(), iterList);
+ }
+
+ // Invalid iterators.
+ @Test
+ public void testConcatNullMetaIterator() {
+ assertThrows("Expect null verification error.", NullPointerException.class,
+ () -> verifyConcat(Collections.emptyIterator(), null));
+ }
+
+ // Concat null iterators.
+ @Test
+ public void testConcatNullElementIterators() {
+ List<Iterable<Integer>> list = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ list.add(() -> null);
+ }
+ verifyConcat(Collections.emptyIterator(), list);
+ }
+
+ private <T> void verifyConcat(Iterator<T> expectedValues, Iterable<Iterable<T>> metaIter) {
+ Iterator<T> iter = Iterables.concat(metaIter).iterator();
+ while (expectedValues.hasNext()) {
+ assertTrue(iter.hasNext());
+ T v1 = expectedValues.next();
+ T v2 = iter.next();
+ assertEquals(v1, v2);
+ }
+ assertFalse(iter.hasNext());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org