You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2013/11/26 16:18:17 UTC
[22/37] git commit: ACCUMULO-804 defend against
FileNotFoundExceptions for hadoop-2.0
ACCUMULO-804 defend against FileNotFoundExceptions for hadoop-2.0
git-svn-id: https://svn.apache.org/repos/asf/accumulo/branches/1.5@1458344 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit aa0ca4a30fcbae39e33fd0ae7e9c6e5dd44c7cd8)
Reason: Hadoop2 Compat
Author: Eric C. Newton <ec...@apache.org>
Ref: ACCUMULO-1792
only one ref to fs.delete() in GarbageCollectWriteAheadLog
Signed-off-by: Eric Newton <er...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1b1334f5
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1b1334f5
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1b1334f5
Branch: refs/heads/1.4.5-SNAPSHOT
Commit: 1b1334f5d26bda6a6be435dd277e3eca225a98c3
Parents: ec2aaa6
Author: Jonathan M Hsieh <jo...@cloudera.com>
Authored: Wed May 29 13:57:46 2013 -0700
Committer: Eric Newton <er...@gmail.com>
Committed: Mon Nov 25 16:06:42 2013 -0500
----------------------------------------------------------------------
.../apache/accumulo/core/client/ZooKeeperInstance.java | 8 +++++++-
.../util/shell/commands/ImportDirectoryCommand.java | 13 ++++++++++---
.../server/gc/GarbageCollectWriteAheadLogs.java | 3 +++
.../accumulo/server/gc/SimpleGarbageCollector.java | 12 +++++++++---
.../accumulo/server/master/tableOps/BulkImport.java | 7 ++++++-
.../apache/accumulo/server/tabletserver/Tablet.java | 10 +++++++++-
6 files changed, 44 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b1334f5/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
index 069d217..fcf8f55 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.core.client;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collections;
@@ -272,7 +273,12 @@ public class ZooKeeperInstance implements Instance {
public static String getInstanceIDFromHdfs(Path instanceDirectory) {
try {
FileSystem fs = FileUtil.getFileSystem(CachedConfiguration.getInstance(), AccumuloConfiguration.getSiteConfiguration());
- FileStatus[] files = fs.listStatus(instanceDirectory);
+ FileStatus[] files = null;
+ try {
+ files = fs.listStatus(instanceDirectory);
+ } catch (FileNotFoundException ex) {
+ // ignored
+ }
log.debug("Trying to read instance id from " + instanceDirectory);
if (files == null || files.length == 0) {
log.error("unable obtain instance id at " + instanceDirectory);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b1334f5/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ImportDirectoryCommand.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ImportDirectoryCommand.java b/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ImportDirectoryCommand.java
index 3deefa7..31479a1 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ImportDirectoryCommand.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ImportDirectoryCommand.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.core.util.shell.commands;
+import java.io.FileNotFoundException;
import java.io.IOException;
import org.apache.accumulo.core.client.AccumuloException;
@@ -45,10 +46,16 @@ public class ImportDirectoryCommand extends Command {
String failureDir = cl.getArgs()[1];
boolean setTime = Boolean.parseBoolean(cl.getArgs()[2]);
- FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
- FileStatus failStatus = fs.getFileStatus(new Path(failureDir));
- if (failStatus == null || !failStatus.isDir() || fs.listStatus(new Path(failureDir)).length != 0)
+ final FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
+ FileStatus failStatus = null;
+ try {
+ failStatus = fs.getFileStatus(new Path(failureDir));
+ } catch (FileNotFoundException ex) {
+ // ignored
+ }
+ if (failStatus == null || !failStatus.isDir() || fs.listStatus(new Path(failureDir)).length != 0) {
throw new AccumuloException(failureDir + " is not an empty directory");
+ }
shellState.getConnector().tableOperations().importDirectory(shellState.getTableName(), dir, failureDir, setTime);
return 0;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b1334f5/src/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java b/src/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java
index 38f00be..06e5349 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/gc/GarbageCollectWriteAheadLogs.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.server.gc;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -138,6 +139,8 @@ public class GarbageCollectWriteAheadLogs {
for (FileStatus match : fs.globStatus(new Path(ServerConstants.getRecoveryDir(), file + "*"))) {
fs.delete(match.getPath(), true);
}
+ } catch (FileNotFoundException ex) {
+ // ignored
} catch (IOException ex) {
log.warn("Error deleting recovery data: ", ex);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b1334f5/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java b/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
index 50bdfed..22c3c0e 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
@@ -355,9 +355,15 @@ public class SimpleGarbageCollector implements Iface {
// tableIdsWithDeletes should now contain the set of deleted tables that had dirs deleted
for (String delTableId : tableIdsWithDeletes) {
- // if dir exist and is empty, then empty list is returned... if dir does not exist
- // then null is returned
- FileStatus[] tabletDirs = fs.listStatus(new Path(ServerConstants.getTablesDir() + "/" + delTableId));
+ // if dir exist and is empty, then empty list is returned...
+ // hadoop 1.0 will return null if the file doesn't exist
+ // hadoop 2.0 will throw an exception if the file does not exist
+ FileStatus[] tabletDirs = null;
+ try {
+ tabletDirs = fs.listStatus(new Path(ServerConstants.getTablesDir() + "/" + delTableId));
+ } catch (FileNotFoundException ex) {
+ // ignored
+ }
if (tabletDirs == null)
continue;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b1334f5/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java b/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java
index 0438563..e9d241c 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/master/tableOps/BulkImport.java
@@ -154,7 +154,12 @@ public class BulkImport extends MasterRepo {
ServerConfiguration.getSiteConfiguration()));
;
Path errorPath = new Path(errorDir);
- FileStatus errorStatus = fs.getFileStatus(errorPath);
+ FileStatus errorStatus = null;
+ try {
+ errorStatus = fs.getFileStatus(errorPath);
+ } catch (FileNotFoundException ex) {
+ // ignored
+ }
if (errorStatus == null)
throw new ThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, errorDir
+ " does not exist");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1b1334f5/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java b/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
index ef4f27a..09903bf 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
@@ -23,6 +23,9 @@ package org.apache.accumulo.server.tabletserver;
*
*/
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -471,7 +474,12 @@ public class Tablet {
private void checkTabletDir(Path tabletDir) throws IOException {
- FileStatus[] files = fs.listStatus(tabletDir);
+ FileStatus[] files = null;
+ try {
+ files = fs.listStatus(tabletDir);
+ } catch (FileNotFoundException ex) {
+ // ignored
+ }
if (files == null) {
if (tabletDir.getName().startsWith("c-"))