svn commit: r1506905 - in /manifoldcf/branches/release-1.3-branch: ./ CHANGES.txt connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java

[kw...@apache.org]

Author: kwright
Date: Thu Jul 25 11:35:39 2013
New Revision: 1506905

URL: http://svn.apache.org/r1506905
Log:
Pull up fix for CONNECTORS-760 from trunk.

Modified:
    manifoldcf/branches/release-1.3-branch/   (props changed)
    manifoldcf/branches/release-1.3-branch/CHANGES.txt
    manifoldcf/branches/release-1.3-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java

Propchange: manifoldcf/branches/release-1.3-branch/
------------------------------------------------------------------------------
  Merged /manifoldcf/trunk:r1506902

Modified: manifoldcf/branches/release-1.3-branch/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.3-branch/CHANGES.txt?rev=1506905&r1=1506904&r2=1506905&view=diff
==============================================================================
--- manifoldcf/branches/release-1.3-branch/CHANGES.txt (original)
+++ manifoldcf/branches/release-1.3-branch/CHANGES.txt Thu Jul 25 11:35:39 2013
@@ -3,6 +3,9 @@ $Id$
 
 ======================= Release 1.3 =====================
 
+CONNECTORS-760: HDFSRepositoryConnector's version string is always start with '-'.
+(Minoru Osuka)
+
 CONNECTORS-759: Fix broken content type for login page.
 (Shinichiro Abe, Karl Wright)
 

Modified: manifoldcf/branches/release-1.3-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.3-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java?rev=1506905&r1=1506904&r2=1506905&view=diff
==============================================================================
--- manifoldcf/branches/release-1.3-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java (original)
+++ manifoldcf/branches/release-1.3-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java Thu Jul 25 11:35:39 2013
@@ -352,7 +352,8 @@ public class HDFSRepositoryConnector ext
             StringBuilder sb = new StringBuilder();
             // Check if the path is to be converted.  We record that info in the version string so that we'll reindex documents whose
             // URI's change.
-            String convertPath = findConvertPath(spec, fileStatus.getPath());
+            String nameNode = "hdfs://" + nameNodeHost + ":" + nameNodePort;
+            String convertPath = findConvertPath(nameNode, spec, fileStatus.getPath());
             if (convertPath != null)
             {
               // Record the path.
@@ -1340,7 +1341,7 @@ public class HDFSRepositoryConnector ext
   *@param documentIdentifier is the document identifier.
   *@return the part of the path to be converted, or null.
   */
-  protected static String findConvertPath(DocumentSpecification spec, Path theFile)
+  protected static String findConvertPath(String nameNode, DocumentSpecification spec, Path theFile)
   {
     String fullpath = theFile.toString();
     for (int j = 0; j < spec.getChildCount(); j++)
@@ -1352,6 +1353,7 @@ public class HDFSRepositoryConnector ext
         String convertToURI = sn.getAttributeValue("converttouri");
         if (path.length() > 0 && convertToURI != null && convertToURI.equals("true"))
         {
+          path = nameNode + path;
           if (!path.endsWith("/"))
             path += "/";
           if (fullpath.startsWith(path))
@@ -1382,13 +1384,6 @@ public class HDFSRepositoryConnector ext
   protected static boolean checkInclude(String nameNode, FileStatus fileStatus, String fileName, DocumentSpecification documentSpecification)
     throws ManifoldCFException
   {
-    /*
-     * TODO:
-     * fileName = hdfs://localhost:9000/user/minoru/KEN_ALL_UTF-8_UNIX_SHRINK.CSV
-     * pathPart = hdfs://localhost:9000/user/minoru
-     * fliePart = KEN_ALL_UTF-8_UNIX_SHRINK.CSV
-     * path = /user/minoru => hdfs://localhost:9000/user/minoru
-     */
     if (Logging.connectors.isDebugEnabled())
     {
       Logging.connectors.debug("Checking whether to include file '"+fileName+"'");