[incubator-pinot] 01/01: Deleting preprocess output post segment creation

[je...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jenniferdai pushed a commit to branch deletepreprocessoutput
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git

commit 24d551c890322ecef1aaa75454ffe903c7a519f4
Author: Jennifer Dai <jd...@linkedin.com>
AuthorDate: Fri Sep 6 10:49:13 2019 -0700

    Deleting preprocess output post segment creation
---
 .../java/org/apache/pinot/hadoop/job/SegmentCreationJob.java   | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java
index aa68f10..5cba226 100644
--- a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java
+++ b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java
@@ -63,6 +63,7 @@ public class SegmentCreationJob extends BaseSegmentJob {
   protected final Path _schemaFile;
   protected final String _defaultPermissionsMask;
   protected final List<PushLocation> _pushLocations;
+  protected Path _preprocessOutputPath = null;
 
   protected FileSystem _fileSystem;
 
@@ -90,6 +91,11 @@ public class SegmentCreationJob extends BaseSegmentJob {
       _pushLocations = null;
     }
 
+    String preprocessOutputPath = _properties.getProperty(JobConfigConstants.PREPROCESS_PATH_TO_OUTPUT);
+    if (preprocessOutputPath != null) {
+      _preprocessOutputPath = getPathFromProperty(preprocessOutputPath);
+    }
+
     _logger.info("*********************************************************************");
     _logger.info("Input Pattern: {}", _inputPattern);
     _logger.info("Output Directory: {}", _outputDir);
@@ -192,6 +198,10 @@ public class SegmentCreationJob extends BaseSegmentJob {
     // Delete the staging directory
     _logger.info("Deleting the staging directory: {}", _stagingDir);
     _fileSystem.delete(_stagingDir, true);
+
+    if (_preprocessOutputPath != null) {
+      _fileSystem.delete(_preprocessOutputPath, true);
+    }
   }
 
   @Nullable


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org