You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by mi...@apache.org on 2014/01/17 04:11:43 UTC
svn commit: r1559000 - in /manifoldcf/trunk: ./ connectors/hdfs/
connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/
connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/
Author: minoru
Date: Fri Jan 17 03:11:43 2014
New Revision: 1559000
URL: http://svn.apache.org/r1559000
Log:
CONNECTORS-858 IO exception: No FileSystem for scheme: hdfs
Modified:
manifoldcf/trunk/build.xml
manifoldcf/trunk/connectors/hdfs/build.xml
manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
manifoldcf/trunk/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/editConfiguration.html
Modified: manifoldcf/trunk/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/build.xml?rev=1559000&r1=1558999&r2=1559000&view=diff
==============================================================================
--- manifoldcf/trunk/build.xml (original)
+++ manifoldcf/trunk/build.xml Fri Jan 17 03:11:43 2014
@@ -3631,6 +3631,12 @@ Use Apache Forrest version forrest-0.9-d
<param name="artifact-name" value="commons-configuration"/>
<param name="artifact-type" value="jar"/>
</antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="commons-cli"/>
+ <param name="artifact-version" value="1.2"/>
+ <param name="artifact-name" value="commons-cli"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
</target>
<target name="download-slf4j">
@@ -4214,13 +4220,31 @@ Use Apache Forrest version forrest-0.9-d
<param name="artifact-name" value="hadoop-auth"/>
<param name="artifact-type" value="jar"/>
</antcall>
+ <antcall target="download-via-maven">
+ <param name="target" value="lib"/>
+ <param name="project-path" value="org/apache/hadoop"/>
+ <param name="artifact-version" value="2.2.0"/>
+ <param name="artifact-name" value="hadoop-hdfs"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
</target>
- <target name="download-guava">
+ <target name="download-protobuf">
+ <mkdir dir="lib"/>
+ <antcall target="download-via-maven">
+ <param name="target" value="lib"/>
+ <param name="project-path" value="com/google/protobuf"/>
+ <param name="artifact-version" value="2.5.0"/>
+ <param name="artifact-name" value="protobuf-java"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ </target>
+
+ <target name="download-guava">
<antcall target="download-via-maven">
<param name="target" value="lib"/>
<param name="project-path" value="com/google/guava"/>
- <param name="artifact-version" value="r09"/>
+ <param name="artifact-version" value="11.0.2"/>
<param name="artifact-name" value="guava"/>
<param name="artifact-type" value="jar"/>
</antcall>
@@ -4248,7 +4272,7 @@ Use Apache Forrest version forrest-0.9-d
</antcall>
</target>
- <target name="make-core-deps" depends="download-jira-client,download-google-api-client,download-dropbox-client,download-solrj,download-zookeeper,download-httpcomponents,download-json,download-hsqldb,download-xerces,download-commons,download-elasticsearch-plugin,download-solr-plugins,download-sharepoint-plugins,download-jstl,download-xmlgraphics-commons,download-wstx-asl,download-xmlsec,download-xml-apis,download-wss4j,download-velocity,download-streambuffer,download-stax,download-servlet-api,download-xml-resolver,download-osgi,download-opensaml,download-mimepull,download-mail,download-log4j,download-junit,download-jaxws,download-glassfish,download-jaxb,download-tomcat,download-h2,download-h2-support,download-geronimo-specs,download-fop,download-derby,download-postgresql,download-axis,download-saaj,download-wsdl4j,download-castor,download-jetty,download-slf4j,download-xalan,download-activation,download-avalon-framework,download-poi,download-chemistry,download-ecj,download-hadoop"
>
+ <target name="make-core-deps" depends="download-jira-client,download-google-api-client,download-dropbox-client,download-solrj,download-zookeeper,download-httpcomponents,download-json,download-hsqldb,download-xerces,download-commons,download-elasticsearch-plugin,download-solr-plugins,download-sharepoint-plugins,download-jstl,download-xmlgraphics-commons,download-wstx-asl,download-xmlsec,download-xml-apis,download-wss4j,download-velocity,download-streambuffer,download-stax,download-servlet-api,download-xml-resolver,download-osgi,download-opensaml,download-mimepull,download-mail,download-log4j,download-junit,download-jaxws,download-glassfish,download-jaxb,download-tomcat,download-h2,download-h2-support,download-geronimo-specs,download-fop,download-derby,download-postgresql,download-axis,download-saaj,download-wsdl4j,download-castor,download-jetty,download-slf4j,download-xalan,download-activation,download-avalon-framework,download-poi,download-chemistry,download-ecj,download-hadoop,
download-protobuf">
<copy todir="lib">
<fileset dir="lib-license" includes="*.txt"/>
</copy>
Modified: manifoldcf/trunk/connectors/hdfs/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/hdfs/build.xml?rev=1559000&r1=1558999&r2=1559000&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/hdfs/build.xml (original)
+++ manifoldcf/trunk/connectors/hdfs/build.xml Fri Jan 17 03:11:43 2014
@@ -22,11 +22,14 @@
<path id="connector-classpath">
<path refid="mcf-connector-build.connector-classpath"/>
<fileset dir="../../lib">
+ <include name="commons-cli*.jar"/>
<include name="commons-configuration*.jar"/>
<include name="hadoop-common*.jar"/>
<include name="hadoop-annotations*.jar"/>
<include name="hadoop-auth*.jar"/>
+ <include name="hadoop-hdfs*.jar"/>
<include name="guava*.jar"/>
+ <include name="protobuf-java*.jar"/>
</fileset>
</path>
@@ -34,11 +37,14 @@
<mkdir dir="dist/lib"/>
<copy todir="dist/lib">
<fileset dir="../../lib">
+ <include name="commons-cli*.jar"/>
<include name="commons-configuration*.jar"/>
<include name="hadoop-common*.jar"/>
<include name="hadoop-annotations*.jar"/>
<include name="hadoop-auth*.jar"/>
+ <include name="hadoop-hdfs*.jar"/>
<include name="guava*.jar"/>
+ <include name="protobuf-java*.jar"/>
</fileset>
</copy>
</target>
Modified: manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java?rev=1559000&r1=1558999&r2=1559000&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java (original)
+++ manifoldcf/trunk/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java Fri Jan 17 03:11:43 2014
@@ -359,7 +359,7 @@ public class HDFSRepositoryConnector ext
StringBuilder sb = new StringBuilder();
// Check if the path is to be converted. We record that info in the version string so that we'll reindex documents whose
// URI's change.
- String nameNode = "hdfs://" + nameNodeHost + ":" + nameNodePort;
+ String nameNode = nameNodeProtocol + "://" + nameNodeHost + ":" + nameNodePort;
String convertPath = findConvertPath(nameNode, spec, fileStatus.getPath());
if (convertPath != null)
{
@@ -633,6 +633,7 @@ public class HDFSRepositoryConnector ext
" <option value=\"file\"" + (nameNodeProtocol.equals("file")?" selected=\"true\"":"") + ">file</option>\n"+
" <option value=\"ftp\"" + (nameNodeProtocol.equals("ftp")?" selected=\"true\"":"") + ">ftp</option>\n"+
" <option value=\"har\"" + (nameNodeProtocol.equals("har")?" selected=\"true\"":"") + ">har</option>\n"+
+" <option value=\"hdfs\"" + (nameNodeProtocol.equals("hdfs")?" selected=\"true\"":"") + ">hdfs</option>\n"+
" <option value=\"s3\"" + (nameNodeProtocol.equals("s3")?" selected=\"true\"":"") + ">s3</option>\n"+
" <option value=\"s3n\"" + (nameNodeProtocol.equals("s3n")?" selected=\"true\"":"") + ">s3n</option>\n"+
" <option value=\"viewfs\"" + (nameNodeProtocol.equals("viewfs")?" selected=\"true\"":"") + ">viewfs</option>\n"+
Modified: manifoldcf/trunk/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/editConfiguration.html
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/editConfiguration.html?rev=1559000&r1=1558999&r2=1559000&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/editConfiguration.html (original)
+++ manifoldcf/trunk/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/editConfiguration.html Fri Jan 17 03:11:43 2014
@@ -38,6 +38,11 @@
#else
<option value="har">har</option>
#end
+ #if($NAMENODEPROTOCOL == 'hdfs')
+ <option value="hdfs" selected="true">hdfs</option>
+ #else
+ <option value="hdfs">hdfs</option>
+ #end
#if($NAMENODEPROTOCOL == 's3')
<option value="s3" selected="true">s3</option>
#else