You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by mi...@apache.org on 2013/06/26 15:00:35 UTC
svn commit: r1496915 - in
/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs:
HDFSRepositoryConnector.java HDFSSession.java
Author: minoru
Date: Wed Jun 26 13:00:35 2013
New Revision: 1496915
URL: http://svn.apache.org/r1496915
Log: (empty)
Modified:
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSSession.java
Modified: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java?rev=1496915&r1=1496914&r2=1496915&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java Wed Jun 26 13:00:35 2013
@@ -431,35 +431,30 @@ public class HDFSRepositoryConnector ext
objt.interrupt();
throw new ManifoldCFException("Interrupted: " + e.getMessage(), e, ManifoldCFException.INTERRUPTED);
}
-
- try {
- FileStatus fileStatus = objt.getResponse();
- if (session.getFileSystem().exists(fileStatus.getPath())) {
- if (fileStatus.isDir()) {
+
+ FileStatus fileStatus = objt.getResponse();
+ if (fileStatus != null) {
+ if (fileStatus.isDir()) {
+ long lastModified = fileStatus.getModificationTime();
+ rval[i] = new Long(lastModified).toString();
+ } else {
+ long fileLength = fileStatus.getLen();
+ if (activities.checkLengthIndexable(fileLength)) {
long lastModified = fileStatus.getModificationTime();
- rval[i] = new Long(lastModified).toString();
- } else {
- long fileLength = fileStatus.getLen();
- if (activities.checkLengthIndexable(fileLength)) {
- long lastModified = fileStatus.getModificationTime();
- StringBuilder sb = new StringBuilder();
- if (filePathToUri) {
- sb.append("+");
- } else {
- sb.append("-");
- }
- sb.append(new Long(lastModified).toString()).append(":").append(new Long(fileLength).toString());
- rval[i] = sb.toString();
+ StringBuilder sb = new StringBuilder();
+ if (filePathToUri) {
+ sb.append("+");
} else {
- rval[i] = null;
+ sb.append("-");
}
+ sb.append(new Long(lastModified).toString()).append(":").append(new Long(fileLength).toString());
+ rval[i] = sb.toString();
+ } else {
+ rval[i] = null;
}
- } else {
- rval[i] = null;
}
- } catch (IOException e) {
- objt.interrupt();
- throw new ManifoldCFException(e);
+ } else {
+ rval[i] = null;
}
}
@@ -507,7 +502,7 @@ public class HDFSRepositoryConnector ext
FileStatus fileStatus = objt.getResponse();
- if (!session.getFileSystem().exists(fileStatus.getPath())) {
+ if (fileStatus == null) {
continue;
}
Modified: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSSession.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSSession.java?rev=1496915&r1=1496914&r2=1496915&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSSession.java (original)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSSession.java Wed Jun 26 13:00:35 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.manifoldcf.core.common.*;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -95,7 +96,11 @@ public class HDFSSession {
}
public FileStatus getObject(String id) throws IOException {
- return fileSystem.getFileStatus(new Path(id));
+ try {
+ return fileSystem.getFileStatus(new Path(id));
+ } catch(FileNotFoundException e) {
+ return null;
+ }
}
public FSDataInputStream getFSDataInputStream(String id) throws IOException {