You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by ab...@apache.org on 2026/03/17 08:39:47 UTC
(gobblin) branch master updated: [GOBBLIN-2256]Optimize manifest discovery with paralleism (#4163)
This is an automated email from the ASF dual-hosted git repository.
abhijain pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 26406a8636 [GOBBLIN-2256]Optimize manifest discovery with paralleism (#4163)
26406a8636 is described below
commit 26406a863688e289063e92bd421bf866f84e5586
Author: pratapaditya04 <th...@gmail.com>
AuthorDate: Tue Mar 17 14:09:41 2026 +0530
[GOBBLIN-2256]Optimize manifest discovery with paralleism (#4163)
* optimizing manifest discovery
---
.../data/management/copy/ManifestBasedDataset.java | 229 +++++++++++++--------
.../dataset/ManifestBasedDatasetFinderTest.java | 105 ++++++++++
2 files changed, 253 insertions(+), 81 deletions(-)
diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/ManifestBasedDataset.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/ManifestBasedDataset.java
index cab65f1275..e25ac0b81f 100644
--- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/ManifestBasedDataset.java
+++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/ManifestBasedDataset.java
@@ -27,6 +27,8 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.fs.FileStatus;
@@ -66,13 +68,20 @@ public class ManifestBasedDataset implements IterableCopyableDataset {
private static final String DELETE_FILE_NOT_EXIST_ON_SOURCE = ManifestBasedDatasetFinder.CONFIG_PREFIX + ".deleteFileNotExistOnSource";
private static final String COMMON_FILES_PARENT = ManifestBasedDatasetFinder.CONFIG_PREFIX + ".commonFilesParent";
private static final String PERMISSION_CACHE_TTL_SECONDS = ManifestBasedDatasetFinder.CONFIG_PREFIX + ".permission.cache.ttl.seconds";
+ public static final String SKIP_PERMISSION_CHECK = ManifestBasedDatasetFinder.CONFIG_PREFIX + ".skipPermissionCheck";
// Enable setting permission post publish to reset permission bits, default is true
private static final String ENABLE_SET_PERMISSION_POST_PUBLISH = ManifestBasedDatasetFinder.CONFIG_PREFIX + ".enableSetPermissionPostPublish";
- public static final String SKIP_PERMISSION_CHECK = ManifestBasedDatasetFinder.CONFIG_PREFIX + ".skipPermissionCheck";
+
+ // Parallelization configuration: when enabled, all manifest entries are loaded upfront and processed concurrently.
+ // Any single file failure propagates immediately (fail-fast), aborting the entire operation.
+ public static final String ENABLE_PARALLEL_PROCESSING = ManifestBasedDatasetFinder.CONFIG_PREFIX + ".enableParallelProcessing";
+
private static final String DEFAULT_PERMISSION_CACHE_TTL_SECONDS = "30";
private static final String DEFAULT_COMMON_FILES_PARENT = "/";
private static final boolean DEFAULT_SKIP_PERMISSION_CHECK = false;
+ private static final boolean DEFAULT_ENABLE_PARALLEL_PROCESSING = true;
+
private final FileSystem srcFs;
private final FileSystem manifestReadFs;
private final Path manifestPath;
@@ -80,9 +89,9 @@ public class ManifestBasedDataset implements IterableCopyableDataset {
private final boolean deleteFileThatNotExistOnSource;
private final String commonFilesParent;
private final int permissionCacheTTLSeconds;
-
private final boolean enableSetPermissionPostPublish;
private final boolean skipPermissionCheck;
+ private final boolean enableParallelProcessing;
public ManifestBasedDataset(final FileSystem srcFs, final FileSystem manifestReadFs, final Path manifestPath, final Properties properties) {
this.srcFs = srcFs;
@@ -94,6 +103,7 @@ public class ManifestBasedDataset implements IterableCopyableDataset {
this.permissionCacheTTLSeconds = Integer.parseInt(properties.getProperty(PERMISSION_CACHE_TTL_SECONDS, DEFAULT_PERMISSION_CACHE_TTL_SECONDS));
this.enableSetPermissionPostPublish = Boolean.parseBoolean(properties.getProperty(ENABLE_SET_PERMISSION_POST_PUBLISH, "true"));
this.skipPermissionCheck = Boolean.parseBoolean(properties.getProperty(SKIP_PERMISSION_CHECK, String.valueOf(DEFAULT_SKIP_PERMISSION_CHECK)));
+ this.enableParallelProcessing = Boolean.parseBoolean(properties.getProperty(ENABLE_PARALLEL_PROCESSING, String.valueOf(DEFAULT_ENABLE_PARALLEL_PROCESSING)));
}
@Override
@@ -115,90 +125,51 @@ public class ManifestBasedDataset implements IterableCopyableDataset {
}
CopyManifest.CopyableUnitIterator manifests = null;
- List<CopyEntity> copyEntities = Lists.newArrayList();
- List<FileStatus> toDelete = Lists.newArrayList();
- // map of paths and permissions sorted by depth of path, so that permissions can be set in order
- Map<String, List<OwnerAndPermission>> ancestorOwnerAndPermissions = new HashMap<>();
- Map<String, List<OwnerAndPermission>> ancestorOwnerAndPermissionsForSetPermissionStep = new HashMap<>();
- Map<String, OwnerAndPermission> existingDirectoryPermissionsForSetPermissionStep = new HashMap<>();
+ // Use thread-safe collections when parallel processing is enabled
+ List<CopyEntity> copyEntities = enableParallelProcessing ? new CopyOnWriteArrayList<>() : Lists.newArrayList();
+ List<FileStatus> toDelete = enableParallelProcessing ? new CopyOnWriteArrayList<>() : Lists.newArrayList();
+ // Map of paths and permissions sorted by depth of path, so that permissions can be set in order
+ Map<String, List<OwnerAndPermission>> ancestorOwnerAndPermissions = enableParallelProcessing ? new ConcurrentHashMap<>() : new HashMap<>();
+ Map<String, List<OwnerAndPermission>> ancestorOwnerAndPermissionsForSetPermissionStep = enableParallelProcessing ? new ConcurrentHashMap<>() : new HashMap<>();
+ Map<String, OwnerAndPermission> existingDirectoryPermissionsForSetPermissionStep = enableParallelProcessing ? new ConcurrentHashMap<>() : new HashMap<>();
TreeMap<String, OwnerAndPermission> flattenedAncestorPermissions = new TreeMap<>(
Comparator.comparingInt((String o) -> o.split("/").length).thenComparing(o -> o));
try {
long startTime = System.currentTimeMillis();
manifests = CopyManifest.getReadIterator(this.manifestReadFs, this.manifestPath);
- Cache<String, OwnerAndPermission> permissionMap = CacheBuilder.newBuilder().expireAfterAccess(permissionCacheTTLSeconds, TimeUnit.SECONDS).build();
- int numFiles = 0;
- while (manifests.hasNext()) {
- numFiles++;
- CopyManifest.CopyableUnit file = manifests.next();
+ Cache<String, OwnerAndPermission> permissionMap =
+ CacheBuilder.newBuilder().expireAfterAccess(permissionCacheTTLSeconds, TimeUnit.SECONDS).build();
+
+ if (enableParallelProcessing) {
//todo: We can use fileSet to partition the data in case of some softbound issue
//todo: After partition, change this to directly return iterator so that we can save time if we meet resources limitation
- Path fileToCopy = new Path(file.fileName);
- if (srcFs.exists(fileToCopy)) {
- boolean existOnTarget = targetFs.exists(fileToCopy);
- if (this.skipPermissionCheck && existOnTarget) {
- // Skip Permission Check for files that already exist in the target when skipPermissionCheck is true
- continue;
- }
- FileStatus srcFile = srcFs.getFileStatus(fileToCopy);
- OwnerAndPermission replicatedPermission = CopyableFile.resolveReplicatedOwnerAndPermission(srcFs, srcFile, configuration);
- if (!existOnTarget || shouldCopy(targetFs, srcFile, targetFs.getFileStatus(fileToCopy), replicatedPermission)) {
- CopyableFile.Builder copyableFileBuilder =
- CopyableFile.fromOriginAndDestination(srcFs, srcFile, fileToCopy, configuration)
- .fileSet(datasetURN())
- .datasetOutputPath(fileToCopy.toString())
- .ancestorsOwnerAndPermission(
- CopyableFile.resolveReplicatedOwnerAndPermissionsRecursivelyWithCache(srcFs, fileToCopy.getParent(),
- new Path(commonFilesParent), configuration, permissionMap))
- .destinationOwnerAndPermission(replicatedPermission);
- CopyableFile copyableFile = copyableFileBuilder.build();
- copyableFile.setFsDatasets(srcFs, targetFs);
- copyEntities.add(copyableFile);
-
- // In case of directory with 000 permission, the permission is changed to 100 due to HadoopUtils::addExecutePermissionToOwner
- // getting called from CopyDataPublisher::preserveFileAttrInPublisher -> FileAwareInputStreamDataWriter::setPathPermission ->
- // FileAwareInputStreamDataWriter::setOwnerExecuteBitIfDirectory -> HadoopUtils::addExecutePermissionToOwner
- // We need to revert this extra permission change in setPermissionStep
- if (srcFile.isDirectory() && !srcFile.getPermission().getUserAction().implies(FsAction.EXECUTE)
- && !ancestorOwnerAndPermissionsForSetPermissionStep.containsKey(PathUtils.getPathWithoutSchemeAndAuthority(fileToCopy).toString())) {
- OwnerAndPermission srcFileOwnerPermissionReplicatedForDest = new OwnerAndPermission(copyableFile.getDestinationOwnerAndPermission());
- if(!targetFs.exists(fileToCopy)) {
- List<OwnerAndPermission> ancestorsOwnerAndPermissionUpdated = new ArrayList<>();
- ancestorsOwnerAndPermissionUpdated.add(srcFileOwnerPermissionReplicatedForDest);
- copyableFile.getAncestorsOwnerAndPermission().forEach(ancestorOwnPerm -> ancestorsOwnerAndPermissionUpdated.add(new OwnerAndPermission(ancestorOwnPerm)));
- ancestorOwnerAndPermissionsForSetPermissionStep.put(fileToCopy.toString(), ancestorsOwnerAndPermissionUpdated);
- }
- else {
- // If the path exists, update only current directory permission in post publish step and not entire hierarchy.
- existingDirectoryPermissionsForSetPermissionStep.put(fileToCopy.toString(), srcFileOwnerPermissionReplicatedForDest);
- }
- }
-
- // Always grab the parent since the above permission setting should be setting the permission for a folder itself
- // {@link CopyDataPublisher#preserveFileAttrInPublisher} will be setting the permission for the empty child dir
- Path fromPath = fileToCopy.getParent();
- // Avoid duplicate calculation for the same ancestor
- if (fromPath != null && !ancestorOwnerAndPermissions.containsKey(PathUtils.getPathWithoutSchemeAndAuthority(fromPath).toString()) && !targetFs.exists(fromPath)) {
- ancestorOwnerAndPermissions.put(fromPath.toString(), copyableFile.getAncestorsOwnerAndPermission());
- if (!ancestorOwnerAndPermissionsForSetPermissionStep.containsKey(PathUtils.getPathWithoutSchemeAndAuthority(fromPath).toString())) {
- ancestorOwnerAndPermissionsForSetPermissionStep.put(fromPath.toString(), copyableFile.getAncestorsOwnerAndPermission());
- }
- }
-
- if (existOnTarget && srcFile.isFile()) {
- // this is to match the existing publishing behavior where we won't rewrite the target when it's already existed
- // todo: Change the publish behavior to support overwrite destination file during rename, instead of relying on this delete step which is needed if we want to support task level publish
- toDelete.add(targetFs.getFileStatus(fileToCopy));
- }
- }
- } else if (deleteFileThatNotExistOnSource && targetFs.exists(fileToCopy)) {
- toDelete.add(targetFs.getFileStatus(fileToCopy));
+ List<CopyManifest.CopyableUnit> allFiles = new ArrayList<>();
+ while (manifests.hasNext()) {
+ allFiles.add(manifests.next());
+ }
+ log.info("Starting parallel processing of {} files from manifest {}", allFiles.size(), this.manifestPath);
+ processFilesInParallel(allFiles, targetFs, configuration, permissionMap, copyEntities, toDelete,
+ ancestorOwnerAndPermissions, ancestorOwnerAndPermissionsForSetPermissionStep,
+ existingDirectoryPermissionsForSetPermissionStep);
+ log.info("Completed parallel processing of {} files in {} ms", allFiles.size(), System.currentTimeMillis() - startTime);
+ } else {
+ //todo: We can use fileSet to partition the data in case of some softbound issue
+ //todo: After partition, change this to directly return iterator so that we can save time if we meet resources limitation
+ int numFiles = 0;
+ while (manifests.hasNext()) {
+ numFiles++;
+ processFile(manifests.next(), targetFs, configuration, permissionMap, copyEntities, toDelete,
+ ancestorOwnerAndPermissions, ancestorOwnerAndPermissionsForSetPermissionStep,
+ existingDirectoryPermissionsForSetPermissionStep);
}
+ log.info("Completed sequential processing of {} files in {} ms", numFiles, System.currentTimeMillis() - startTime);
}
// Precreate the directories to avoid an edge case where recursive rename can create extra directories in the target
- CommitStep createDirectoryWithPermissionsCommitStep = new CreateDirectoryWithPermissionsCommitStep(targetFs, ancestorOwnerAndPermissions, this.properties);
- copyEntities.add(new PrePublishStep(datasetURN(), Maps.newHashMap(), createDirectoryWithPermissionsCommitStep, 1));
+ CommitStep createDirectoryWithPermissionsCommitStep =
+ new CreateDirectoryWithPermissionsCommitStep(targetFs, ancestorOwnerAndPermissions, this.properties);
+ copyEntities.add(
+ new PrePublishStep(datasetURN(), Maps.newHashMap(), createDirectoryWithPermissionsCommitStep, 1));
if (this.enableSetPermissionPostPublish) {
for (Map.Entry<String, List<OwnerAndPermission>> recursiveParentPermissions : ancestorOwnerAndPermissionsForSetPermissionStep.entrySet()) {
@@ -216,7 +187,8 @@ public class ManifestBasedDataset implements IterableCopyableDataset {
Path currentPath = new Path(existingDirectoryPermissions.getKey());
flattenedAncestorPermissions.put(currentPath.toString(), existingDirectoryPermissions.getValue());
}
- CommitStep setPermissionCommitStep = new SetPermissionCommitStep(targetFs, flattenedAncestorPermissions, this.properties);
+ CommitStep setPermissionCommitStep =
+ new SetPermissionCommitStep(targetFs, flattenedAncestorPermissions, this.properties);
copyEntities.add(new PostPublishStep(datasetURN(), Maps.newHashMap(), setPermissionCommitStep, 1));
}
if (!toDelete.isEmpty()) {
@@ -224,19 +196,24 @@ public class ManifestBasedDataset implements IterableCopyableDataset {
CommitStep step = new DeleteFileCommitStep(targetFs, toDelete, this.properties, Optional.<Path>absent());
copyEntities.add(new PrePublishStep(datasetURN(), Maps.newHashMap(), step, 1));
}
- log.info(String.format("Workunits calculation took %s milliseconds to process %s files", System.currentTimeMillis() - startTime, numFiles));
} catch (JsonIOException | JsonSyntaxException e) {
//todo: update error message to point to a sample json file instead of schema which is hard to understand
- log.warn(String.format("Failed to read Manifest path %s on filesystem %s, please make sure it's in correct json format with schema"
- + " {type:array, items:{type: object, properties:{id:{type:String}, fileName:{type:String}, fileGroup:{type:String}, fileSizeInBytes: {type:Long}}}}",
+ log.warn(String.format(
+ "Failed to read Manifest path %s on filesystem %s, please make sure it's in correct json format with schema"
+ + " {type:array, items:{type: object, properties:{id:{type:String}, fileName:{type:String}, fileGroup:{type:String}, fileSizeInBytes: {type:Long}}}}",
manifestPath.toString(), manifestReadFs.getUri().toString()), e);
throw new IOException(e);
} catch (Exception e) {
- log.warn(String.format("Failed to process Manifest path %s on filesystem %s, due to", manifestPath.toString(), manifestReadFs.getUri().toString()), e);
+ log.warn(String.format("Failed to process Manifest path %s on filesystem %s, due to", manifestPath.toString(),
+ manifestReadFs.getUri().toString()), e);
throw new IOException(e);
} finally {
if (manifests != null) {
- manifests.close();
+ try {
+ manifests.close();
+ } catch (Exception e) {
+ log.warn("Failed to close manifest iterator for path {}", manifestPath, e);
+ }
}
}
return Collections.singleton(new FileSet.Builder<>(datasetURN(), this).add(copyEntities).build()).iterator();
@@ -247,4 +224,94 @@ public class ManifestBasedDataset implements IterableCopyableDataset {
// Copy only if source is newer than target or if the owner or permission is different
return fileInSource.getModificationTime() > fileInTarget.getModificationTime() || !replicatedPermission.hasSameOwnerAndPermission(targetFs, fileInTarget);
}
+
+ /**
+ * Processes all files concurrently using a parallel stream. Any single file failure propagates immediately
+ * (fail-fast), causing the entire operation to abort with an exception.
+ */
+ private void processFilesInParallel(List<CopyManifest.CopyableUnit> allFiles, FileSystem targetFs,
+ CopyConfiguration configuration, Cache<String, OwnerAndPermission> permissionMap,
+ List<CopyEntity> copyEntities, List<FileStatus> toDelete,
+ Map<String, List<OwnerAndPermission>> ancestorOwnerAndPermissions,
+ Map<String, List<OwnerAndPermission>> ancestorOwnerAndPermissionsForSetPermissionStep,
+ Map<String, OwnerAndPermission> existingDirectoryPermissionsForSetPermissionStep) {
+ allFiles.parallelStream().forEach(file -> {
+ try {
+ processFile(file, targetFs, configuration, permissionMap, copyEntities, toDelete,
+ ancestorOwnerAndPermissions, ancestorOwnerAndPermissionsForSetPermissionStep,
+ existingDirectoryPermissionsForSetPermissionStep);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to process file: " + file.fileName, e);
+ }
+ });
+ }
+
+ private void processFile(CopyManifest.CopyableUnit file, FileSystem targetFs,
+ CopyConfiguration configuration, Cache<String, OwnerAndPermission> permissionMap,
+ List<CopyEntity> copyEntities, List<FileStatus> toDelete,
+ Map<String, List<OwnerAndPermission>> ancestorOwnerAndPermissions,
+ Map<String, List<OwnerAndPermission>> ancestorOwnerAndPermissionsForSetPermissionStep,
+ Map<String, OwnerAndPermission> existingDirectoryPermissionsForSetPermissionStep) throws Exception {
+ Path fileToCopy = new Path(file.fileName);
+ if (srcFs.exists(fileToCopy)) {
+ boolean existOnTarget = targetFs.exists(fileToCopy);
+ if (this.skipPermissionCheck && existOnTarget) {
+ // Skip permission check for files that already exist in the target when skipPermissionCheck is true
+ return;
+ }
+ FileStatus srcFile = srcFs.getFileStatus(fileToCopy);
+ OwnerAndPermission replicatedPermission = CopyableFile.resolveReplicatedOwnerAndPermission(srcFs, srcFile, configuration);
+ if (!existOnTarget || shouldCopy(targetFs, srcFile, targetFs.getFileStatus(fileToCopy), replicatedPermission)) {
+ CopyableFile copyableFile = CopyableFile.fromOriginAndDestination(srcFs, srcFile, fileToCopy, configuration)
+ .fileSet(datasetURN())
+ .datasetOutputPath(fileToCopy.toString())
+ .ancestorsOwnerAndPermission(
+ CopyableFile.resolveReplicatedOwnerAndPermissionsRecursivelyWithCache(srcFs, fileToCopy.getParent(),
+ new Path(commonFilesParent), configuration, permissionMap))
+ .destinationOwnerAndPermission(replicatedPermission)
+ .build();
+ copyableFile.setFsDatasets(srcFs, targetFs);
+ copyEntities.add(copyableFile);
+
+ // In case of directory with 000 permission, the permission is changed to 100 due to HadoopUtils::addExecutePermissionToOwner
+ // getting called from CopyDataPublisher::preserveFileAttrInPublisher -> FileAwareInputStreamDataWriter::setPathPermission ->
+ // FileAwareInputStreamDataWriter::setOwnerExecuteBitIfDirectory -> HadoopUtils::addExecutePermissionToOwner
+ // We need to revert this extra permission change in setPermissionStep
+ if (srcFile.isDirectory() && !srcFile.getPermission().getUserAction().implies(FsAction.EXECUTE)
+ && !ancestorOwnerAndPermissionsForSetPermissionStep.containsKey(PathUtils.getPathWithoutSchemeAndAuthority(fileToCopy).toString())) {
+ OwnerAndPermission srcFileOwnerPermissionReplicatedForDest = new OwnerAndPermission(copyableFile.getDestinationOwnerAndPermission());
+ if (!targetFs.exists(fileToCopy)) {
+ List<OwnerAndPermission> ancestorsOwnerAndPermissionUpdated = new ArrayList<>();
+ ancestorsOwnerAndPermissionUpdated.add(srcFileOwnerPermissionReplicatedForDest);
+ copyableFile.getAncestorsOwnerAndPermission().forEach(ancestorOwnPerm -> ancestorsOwnerAndPermissionUpdated.add(new OwnerAndPermission(ancestorOwnPerm)));
+ ancestorOwnerAndPermissionsForSetPermissionStep.put(fileToCopy.toString(), ancestorsOwnerAndPermissionUpdated);
+ } else {
+ // If the path exists, update only current directory permission in post publish step and not entire hierarchy.
+ existingDirectoryPermissionsForSetPermissionStep.put(fileToCopy.toString(), srcFileOwnerPermissionReplicatedForDest);
+ }
+ }
+
+ // Always grab the parent since the above permission setting should be setting the permission for a folder itself
+ // {@link CopyDataPublisher#preserveFileAttrInPublisher} will be setting the permission for the empty child dir
+ Path fromPath = fileToCopy.getParent();
+ // Avoid duplicate calculation for the same ancestor
+ if (fromPath != null
+ && !ancestorOwnerAndPermissions.containsKey(PathUtils.getPathWithoutSchemeAndAuthority(fromPath).toString())
+ && !targetFs.exists(fromPath)) {
+ ancestorOwnerAndPermissions.put(fromPath.toString(), copyableFile.getAncestorsOwnerAndPermission());
+ if (!ancestorOwnerAndPermissionsForSetPermissionStep.containsKey(PathUtils.getPathWithoutSchemeAndAuthority(fromPath).toString())) {
+ ancestorOwnerAndPermissionsForSetPermissionStep.put(fromPath.toString(), copyableFile.getAncestorsOwnerAndPermission());
+ }
+ }
+
+ if (existOnTarget && srcFile.isFile()) {
+ // this is to match the existing publishing behavior where we won't rewrite the target when it's already existed
+ // todo: Change the publish behavior to support overwrite destination file during rename, instead of relying on this delete step which is needed if we want to support task level publish
+ toDelete.add(targetFs.getFileStatus(fileToCopy));
+ }
+ }
+ } else if (deleteFileThatNotExistOnSource && targetFs.exists(fileToCopy)) {
+ toDelete.add(targetFs.getFileStatus(fileToCopy));
+ }
+ }
}
diff --git a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/dataset/ManifestBasedDatasetFinderTest.java b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/dataset/ManifestBasedDatasetFinderTest.java
index 2b62ccad8f..dfe0595c70 100644
--- a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/dataset/ManifestBasedDatasetFinderTest.java
+++ b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/dataset/ManifestBasedDatasetFinderTest.java
@@ -570,6 +570,111 @@ public class ManifestBasedDatasetFinderTest {
}
}
+ /**
+ * Verifies that parallel processing produces the same number of copy entities as the default behavior.
+ */
+ @Test
+ public void testParallelProcessingProducesSameResults() throws Exception {
+ Path manifestPath = new Path(getClass().getClassLoader().getResource("manifestBasedDistcpTest/sampleManifest.json").getPath());
+ Properties props = new Properties();
+ props.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/");
+ props.setProperty(ManifestBasedDataset.ENABLE_PARALLEL_PROCESSING, "true");
+
+ try (FileSystem sourceFs = Mockito.mock(FileSystem.class);
+ FileSystem manifestReadFs = Mockito.mock(FileSystem.class);
+ FileSystem destFs = Mockito.mock(FileSystem.class)) {
+ setSourceAndDestFsMocks(sourceFs, destFs, manifestPath, manifestReadFs, true);
+
+ Iterator<FileSet<CopyEntity>> fileSets =
+ new ManifestBasedDataset(sourceFs, manifestReadFs, manifestPath, props).getFileSetIterator(destFs,
+ CopyConfiguration.builder(destFs, props).build());
+ Assert.assertTrue(fileSets.hasNext());
+ FileSet<CopyEntity> fileSet = fileSets.next();
+ Assert.assertEquals(fileSet.getFiles().size(), 4); // 2 files to copy + 1 pre publish step + 1 post publish step
+ }
+ }
+
+ /**
+ * Verifies that sequential processing (parallel disabled) produces the same copy entities.
+ */
+ @Test
+ public void testSequentialProcessingProducesSameResults() throws Exception {
+ Path manifestPath = new Path(getClass().getClassLoader().getResource("manifestBasedDistcpTest/sampleManifest.json").getPath());
+ Properties props = new Properties();
+ props.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/");
+ props.setProperty(ManifestBasedDataset.ENABLE_PARALLEL_PROCESSING, "false");
+
+ try (FileSystem sourceFs = Mockito.mock(FileSystem.class);
+ FileSystem manifestReadFs = Mockito.mock(FileSystem.class);
+ FileSystem destFs = Mockito.mock(FileSystem.class)) {
+ setSourceAndDestFsMocks(sourceFs, destFs, manifestPath, manifestReadFs, true);
+
+ Iterator<FileSet<CopyEntity>> fileSets =
+ new ManifestBasedDataset(sourceFs, manifestReadFs, manifestPath, props).getFileSetIterator(destFs,
+ CopyConfiguration.builder(destFs, props).build());
+ Assert.assertTrue(fileSets.hasNext());
+ FileSet<CopyEntity> fileSet = fileSets.next();
+ Assert.assertEquals(fileSet.getFiles().size(), 4); // 2 files to copy + 1 pre publish step + 1 post publish step
+ Assert.assertTrue(((PrePublishStep) fileSet.getFiles().get(2)).getStep() instanceof CreateDirectoryWithPermissionsCommitStep);
+ }
+ }
+
+ /**
+ * Verifies fail-fast behavior in parallel mode: a single file failure causes the entire operation to throw IOException.
+ */
+ @Test
+ public void testParallelProcessingFailsFastOnError() throws Exception {
+ Path manifestPath = new Path(getClass().getClassLoader().getResource("manifestBasedDistcpTest/sampleManifest.json").getPath());
+ Properties props = new Properties();
+ props.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/");
+ props.setProperty(ManifestBasedDataset.ENABLE_PARALLEL_PROCESSING, "true");
+
+ try (FileSystem sourceFs = Mockito.mock(FileSystem.class);
+ FileSystem manifestReadFs = Mockito.mock(FileSystem.class);
+ FileSystem destFs = Mockito.mock(FileSystem.class)) {
+ setSourceAndDestFsMocks(sourceFs, destFs, manifestPath, manifestReadFs, true);
+ // Inject a failure for one specific file
+ Mockito.when(sourceFs.getFileStatus(new Path("/tmp/dataset/test1.txt")))
+ .thenThrow(new IOException("Simulated IO failure for test1.txt"));
+
+ try {
+ new ManifestBasedDataset(sourceFs, manifestReadFs, manifestPath, props)
+ .getFileSetIterator(destFs, CopyConfiguration.builder(destFs, props).build());
+ Assert.fail("Expected IOException to be thrown due to file processing failure");
+ } catch (IOException e) {
+ // Expected: the failure propagated and aborted the operation
+ }
+ }
+ }
+
+ /**
+ * Verifies fail-fast behavior in sequential mode: a single file failure causes the entire operation to throw IOException.
+ */
+ @Test
+ public void testSequentialProcessingFailsFastOnError() throws Exception {
+ Path manifestPath = new Path(getClass().getClassLoader().getResource("manifestBasedDistcpTest/sampleManifest.json").getPath());
+ Properties props = new Properties();
+ props.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/");
+ props.setProperty(ManifestBasedDataset.ENABLE_PARALLEL_PROCESSING, "false");
+
+ try (FileSystem sourceFs = Mockito.mock(FileSystem.class);
+ FileSystem manifestReadFs = Mockito.mock(FileSystem.class);
+ FileSystem destFs = Mockito.mock(FileSystem.class)) {
+ setSourceAndDestFsMocks(sourceFs, destFs, manifestPath, manifestReadFs, true);
+ // Inject a failure for one specific file
+ Mockito.when(sourceFs.getFileStatus(new Path("/tmp/dataset/test1.txt")))
+ .thenThrow(new IOException("Simulated IO failure for test1.txt"));
+
+ try {
+ new ManifestBasedDataset(sourceFs, manifestReadFs, manifestPath, props)
+ .getFileSetIterator(destFs, CopyConfiguration.builder(destFs, props).build());
+ Assert.fail("Expected IOException to be thrown due to file processing failure");
+ } catch (IOException e) {
+ // Expected: the failure propagated and aborted the operation
+ }
+ }
+ }
+
private void setSourceAndDestFsMocks(FileSystem sourceFs, FileSystem destFs, Path manifestPath, FileSystem manifestReadFs, boolean setFileStatusMock) throws IOException, URISyntaxException {
URI SRC_FS_URI = new URI("source", "the.source.org", "/", null);
URI MANIFEST_READ_FS_URI = new URI("manifest-read", "the.manifest-source.org", "/", null);