You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by am...@apache.org on 2018/01/25 00:29:12 UTC
drill git commit: DRILL-6100: Intermittent failure while reading
Parquet file footer during planning phase
Repository: drill
Updated Branches:
refs/heads/master e791ed62b -> 9926eda21
DRILL-6100: Intermittent failure while reading Parquet file footer during planning phase
close apache/drill#1097
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/9926eda2
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/9926eda2
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/9926eda2
Branch: refs/heads/master
Commit: 9926eda21c748e96d67bce341a76dac3114002af
Parents: e791ed6
Author: Sorabh Hamirwasia <sh...@maprtech.com>
Authored: Sun Jan 21 16:30:25 2018 -0800
Committer: Aman Sinha <as...@maprtech.com>
Committed: Wed Jan 24 13:12:31 2018 -0800
----------------------------------------------------------------------
.../drill/exec/store/parquet/Metadata.java | 19 +++++++++++++++++--
1 file changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/9926eda2/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
index a611c6f..ab3bc19 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.store.parquet;
import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -41,6 +42,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.parquet.column.statistics.Statistics;
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.metadata.BlockMetaData;
@@ -416,8 +418,21 @@ public class Metadata {
* Get the metadata for a single file
*/
private ParquetFileMetadata_v3 getParquetFileMetadata_v3(ParquetTableMetadata_v3 parquetTableMetadata,
- FileStatus file) throws IOException {
- ParquetMetadata metadata = ParquetFileReader.readFooter(fs.getConf(), file);
+ final FileStatus file) throws IOException, InterruptedException {
+ final ParquetMetadata metadata;
+ final UserGroupInformation processUserUgi = ImpersonationUtil.getProcessUserUGI();
+ try {
+ metadata = processUserUgi.doAs(new PrivilegedExceptionAction<ParquetMetadata>() {
+ public ParquetMetadata run() throws Exception {
+ return ParquetFileReader.readFooter(fs.getConf(), file);
+ }
+ });
+ } catch(Exception e) {
+ logger.error("Exception while reading footer of parquet file [Details - path: {}, owner: {}] as process user {}",
+ file.getPath(), file.getOwner(), processUserUgi.getShortUserName(), e);
+ throw e;
+ }
+
MessageType schema = metadata.getFileMetaData().getSchema();
// Map<SchemaPath, OriginalType> originalTypeMap = Maps.newHashMap();