You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by mi...@apache.org on 2013/07/25 13:33:21 UTC
svn commit: r1506902 - in /manifoldcf/trunk: CHANGES.txt
connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
Author: minoru
Date: Thu Jul 25 11:33:21 2013
New Revision: 1506902
URL: http://svn.apache.org/r1506902
Log:
fix CONNECTORS-760.
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1506902&r1=1506901&r2=1506902&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Thu Jul 25 11:33:21 2013
@@ -5,6 +5,9 @@ $Id$
======================= Release 1.3 =====================
+CONNECTORS-760: HDFSRepositoryConnector's version string is always start with '-'.
+(Minoru Osuka)
+
CONNECTORS-759: Fix broken content type for login page.
(Shinichiro Abe, Karl Wright)
Modified: manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java?rev=1506902&r1=1506901&r2=1506902&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java (original)
+++ manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java Thu Jul 25 11:33:21 2013
@@ -352,7 +352,8 @@ public class HDFSRepositoryConnector ext
StringBuilder sb = new StringBuilder();
// Check if the path is to be converted. We record that info in the version string so that we'll reindex documents whose
// URI's change.
- String convertPath = findConvertPath(spec, fileStatus.getPath());
+ String nameNode = "hdfs://" + nameNodeHost + ":" + nameNodePort;
+ String convertPath = findConvertPath(nameNode, spec, fileStatus.getPath());
if (convertPath != null)
{
// Record the path.
@@ -1340,7 +1341,7 @@ public class HDFSRepositoryConnector ext
*@param documentIdentifier is the document identifier.
*@return the part of the path to be converted, or null.
*/
- protected static String findConvertPath(DocumentSpecification spec, Path theFile)
+ protected static String findConvertPath(String nameNode, DocumentSpecification spec, Path theFile)
{
String fullpath = theFile.toString();
for (int j = 0; j < spec.getChildCount(); j++)
@@ -1352,6 +1353,7 @@ public class HDFSRepositoryConnector ext
String convertToURI = sn.getAttributeValue("converttouri");
if (path.length() > 0 && convertToURI != null && convertToURI.equals("true"))
{
+ path = nameNode + path;
if (!path.endsWith("/"))
path += "/";
if (fullpath.startsWith(path))
@@ -1382,13 +1384,6 @@ public class HDFSRepositoryConnector ext
protected static boolean checkInclude(String nameNode, FileStatus fileStatus, String fileName, DocumentSpecification documentSpecification)
throws ManifoldCFException
{
- /*
- * TODO:
- * fileName = hdfs://localhost:9000/user/minoru/KEN_ALL_UTF-8_UNIX_SHRINK.CSV
- * pathPart = hdfs://localhost:9000/user/minoru
- * fliePart = KEN_ALL_UTF-8_UNIX_SHRINK.CSV
- * path = /user/minoru => hdfs://localhost:9000/user/minoru
- */
if (Logging.connectors.isDebugEnabled())
{
Logging.connectors.debug("Checking whether to include file '"+fileName+"'");