You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2019/10/01 04:48:33 UTC
[incubator-pinot] 01/01: more logging
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch bugfixing_hadoop_filesystem
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 2f50cbf7d04f4ef742b644e538404b3e828bcb52
Author: Xiang Fu <xi...@alt-chain.io>
AuthorDate: Mon Sep 30 10:37:50 2019 -0700
more logging
---
.../src/main/java/org/apache/pinot/hadoop/job/BaseSegmentJob.java | 8 +++++++-
.../main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java | 4 ++--
2 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/BaseSegmentJob.java b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/BaseSegmentJob.java
index 7828fdf..fb39ac1 100644
--- a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/BaseSegmentJob.java
+++ b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/BaseSegmentJob.java
@@ -62,7 +62,13 @@ public abstract class BaseSegmentJob extends Configured {
throws IOException {
List<Path> tarFilePaths = new ArrayList<>();
FileSystem fileSystem = FileSystem.get(pathPattern.toUri(), _conf);
- getDataFilePathsHelper(fileSystem, fileSystem.globStatus(pathPattern), tarFilePaths);
+ _logger.info("Using filesystem: {}", fileSystem);
+ final FileStatus[] fileStatuses = fileSystem.globStatus(pathPattern);
+ if (fileStatuses == null) {
+ _logger.warn("Unable to match file status from file path pattern: {}", pathPattern);
+ } else {
+ getDataFilePathsHelper(fileSystem, fileStatuses, tarFilePaths);
+ }
return tarFilePaths;
}
diff --git a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java
index aa68f10..4b6ef68 100644
--- a/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java
+++ b/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java
@@ -109,7 +109,7 @@ public class SegmentCreationJob extends BaseSegmentJob {
return true;
}
return fileName.endsWith(".avro") || fileName.endsWith(".csv") || fileName.endsWith(".json") || fileName
- .endsWith(".thrift");
+ .endsWith(".thrift") || fileName.endsWith(".parquet");
}
public void run()
@@ -117,7 +117,7 @@ public class SegmentCreationJob extends BaseSegmentJob {
_logger.info("Starting {}", getClass().getSimpleName());
// Initialize all directories
- _fileSystem = FileSystem.get(_conf);
+ _fileSystem = FileSystem.get(_outputDir.toUri(), _conf);
JobPreparationHelper.mkdirs(_fileSystem, _outputDir, _defaultPermissionsMask);
JobPreparationHelper.mkdirs(_fileSystem, _stagingDir, _defaultPermissionsMask);
Path stagingInputDir = new Path(_stagingDir, "input");
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org