You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/01/17 04:27:37 UTC
svn commit: r1559005 - in /manifoldcf/branches/release-1.5-branch: ./
connectors/hdfs/
connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/
connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hd...
Author: kwright
Date: Fri Jan 17 03:27:36 2014
New Revision: 1559005
URL: http://svn.apache.org/r1559005
Log:
Pull up hdfs support from trunk - part of CONNECTORS-858.
Modified:
manifoldcf/branches/release-1.5-branch/ (props changed)
manifoldcf/branches/release-1.5-branch/build.xml
manifoldcf/branches/release-1.5-branch/connectors/hdfs/build.xml
manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/ParameterEnum.java
manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/editConfiguration.html
Propchange: manifoldcf/branches/release-1.5-branch/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk:r1559000,1559003
Modified: manifoldcf/branches/release-1.5-branch/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.5-branch/build.xml?rev=1559005&r1=1559004&r2=1559005&view=diff
==============================================================================
--- manifoldcf/branches/release-1.5-branch/build.xml (original)
+++ manifoldcf/branches/release-1.5-branch/build.xml Fri Jan 17 03:27:36 2014
@@ -3631,6 +3631,12 @@ Use Apache Forrest version forrest-0.9-d
<param name="artifact-name" value="commons-configuration"/>
<param name="artifact-type" value="jar"/>
</antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="commons-cli"/>
+ <param name="artifact-version" value="1.2"/>
+ <param name="artifact-name" value="commons-cli"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
</target>
<target name="download-slf4j">
@@ -4214,13 +4220,31 @@ Use Apache Forrest version forrest-0.9-d
<param name="artifact-name" value="hadoop-auth"/>
<param name="artifact-type" value="jar"/>
</antcall>
+ <antcall target="download-via-maven">
+ <param name="target" value="lib"/>
+ <param name="project-path" value="org/apache/hadoop"/>
+ <param name="artifact-version" value="2.2.0"/>
+ <param name="artifact-name" value="hadoop-hdfs"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
</target>
- <target name="download-guava">
+ <target name="download-protobuf">
+ <mkdir dir="lib"/>
+ <antcall target="download-via-maven">
+ <param name="target" value="lib"/>
+ <param name="project-path" value="com/google/protobuf"/>
+ <param name="artifact-version" value="2.5.0"/>
+ <param name="artifact-name" value="protobuf-java"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ </target>
+
+ <target name="download-guava">
<antcall target="download-via-maven">
<param name="target" value="lib"/>
<param name="project-path" value="com/google/guava"/>
- <param name="artifact-version" value="r09"/>
+ <param name="artifact-version" value="11.0.2"/>
<param name="artifact-name" value="guava"/>
<param name="artifact-type" value="jar"/>
</antcall>
@@ -4248,7 +4272,7 @@ Use Apache Forrest version forrest-0.9-d
</antcall>
</target>
- <target name="make-core-deps" depends="download-jira-client,download-google-api-client,download-dropbox-client,download-solrj,download-zookeeper,download-httpcomponents,download-json,download-hsqldb,download-xerces,download-commons,download-elasticsearch-plugin,download-solr-plugins,download-sharepoint-plugins,download-jstl,download-xmlgraphics-commons,download-wstx-asl,download-xmlsec,download-xml-apis,download-wss4j,download-velocity,download-streambuffer,download-stax,download-servlet-api,download-xml-resolver,download-osgi,download-opensaml,download-mimepull,download-mail,download-log4j,download-junit,download-jaxws,download-glassfish,download-jaxb,download-tomcat,download-h2,download-h2-support,download-geronimo-specs,download-fop,download-derby,download-postgresql,download-axis,download-saaj,download-wsdl4j,download-castor,download-jetty,download-slf4j,download-xalan,download-activation,download-avalon-framework,download-poi,download-chemistry,download-ecj,download-hadoop"
>
+ <target name="make-core-deps" depends="download-jira-client,download-google-api-client,download-dropbox-client,download-solrj,download-zookeeper,download-httpcomponents,download-json,download-hsqldb,download-xerces,download-commons,download-elasticsearch-plugin,download-solr-plugins,download-sharepoint-plugins,download-jstl,download-xmlgraphics-commons,download-wstx-asl,download-xmlsec,download-xml-apis,download-wss4j,download-velocity,download-streambuffer,download-stax,download-servlet-api,download-xml-resolver,download-osgi,download-opensaml,download-mimepull,download-mail,download-log4j,download-junit,download-jaxws,download-glassfish,download-jaxb,download-tomcat,download-h2,download-h2-support,download-geronimo-specs,download-fop,download-derby,download-postgresql,download-axis,download-saaj,download-wsdl4j,download-castor,download-jetty,download-slf4j,download-xalan,download-activation,download-avalon-framework,download-poi,download-chemistry,download-ecj,download-hadoop,
download-protobuf">
<copy todir="lib">
<fileset dir="lib-license" includes="*.txt"/>
</copy>
Modified: manifoldcf/branches/release-1.5-branch/connectors/hdfs/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.5-branch/connectors/hdfs/build.xml?rev=1559005&r1=1559004&r2=1559005&view=diff
==============================================================================
--- manifoldcf/branches/release-1.5-branch/connectors/hdfs/build.xml (original)
+++ manifoldcf/branches/release-1.5-branch/connectors/hdfs/build.xml Fri Jan 17 03:27:36 2014
@@ -22,11 +22,14 @@
<path id="connector-classpath">
<path refid="mcf-connector-build.connector-classpath"/>
<fileset dir="../../lib">
+ <include name="commons-cli*.jar"/>
<include name="commons-configuration*.jar"/>
<include name="hadoop-common*.jar"/>
<include name="hadoop-annotations*.jar"/>
<include name="hadoop-auth*.jar"/>
+ <include name="hadoop-hdfs*.jar"/>
<include name="guava*.jar"/>
+ <include name="protobuf-java*.jar"/>
</fileset>
</path>
@@ -34,11 +37,14 @@
<mkdir dir="dist/lib"/>
<copy todir="dist/lib">
<fileset dir="../../lib">
+ <include name="commons-cli*.jar"/>
<include name="commons-configuration*.jar"/>
<include name="hadoop-common*.jar"/>
<include name="hadoop-annotations*.jar"/>
<include name="hadoop-auth*.jar"/>
+ <include name="hadoop-hdfs*.jar"/>
<include name="guava*.jar"/>
+ <include name="protobuf-java*.jar"/>
</fileset>
</copy>
</target>
Modified: manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java?rev=1559005&r1=1559004&r2=1559005&view=diff
==============================================================================
--- manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java (original)
+++ manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java Fri Jan 17 03:27:36 2014
@@ -115,7 +115,7 @@ public class HDFSOutputConnector extends
super.connect(configParams);
nameNodeProtocol = configParams.getParameter(ParameterEnum.namenodeprotocol.name());
if (nameNodeProtocol == null)
- nameNodeProtocol = "file";
+ nameNodeProtocol = "hdfs";
nameNodeHost = configParams.getParameter(ParameterEnum.namenodehost.name());
nameNodePort = configParams.getParameter(ParameterEnum.namenodeport.name());
user = configParams.getParameter(ParameterEnum.user.name());
@@ -182,7 +182,7 @@ public class HDFSOutputConnector extends
protected HDFSSession getSession() throws ManifoldCFException, ServiceInterruption {
if (session == null) {
if (nameNodeProtocol == null)
- nameNodeProtocol = "file";
+ nameNodeProtocol = "hdfs";
if (nameNodeHost == null)
throw new ManifoldCFException("Namenodehost must be specified");
Modified: manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/ParameterEnum.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/ParameterEnum.java?rev=1559005&r1=1559004&r2=1559005&view=diff
==============================================================================
--- manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/ParameterEnum.java (original)
+++ manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/ParameterEnum.java Fri Jan 17 03:27:36 2014
@@ -24,7 +24,7 @@ import java.util.Map;
/** Parameters constants */
public enum ParameterEnum {
- namenodeprotocol("file"),
+ namenodeprotocol("hdfs"),
namenodehost("localhost"),
namenodeport("9000"),
user(""),
Modified: manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java?rev=1559005&r1=1559004&r2=1559005&view=diff
==============================================================================
--- manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java (original)
+++ manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java Fri Jan 17 03:27:36 2014
@@ -123,7 +123,7 @@ public class HDFSRepositoryConnector ext
nameNodeProtocol = configParams.getParameter("namenodeprotocol");
if (nameNodeProtocol == null)
- nameNodeProtocol = "file";
+ nameNodeProtocol = "hdfs";
nameNodeHost = configParams.getParameter("namenodehost");
nameNodePort = configParams.getParameter("namenodeport");
user = configParams.getParameter("user");
@@ -359,7 +359,7 @@ public class HDFSRepositoryConnector ext
StringBuilder sb = new StringBuilder();
// Check if the path is to be converted. We record that info in the version string so that we'll reindex documents whose
// URI's change.
- String nameNode = "hdfs://" + nameNodeHost + ":" + nameNodePort;
+ String nameNode = nameNodeProtocol + "://" + nameNodeHost + ":" + nameNodePort;
String convertPath = findConvertPath(nameNode, spec, fileStatus.getPath());
if (convertPath != null)
{
@@ -604,7 +604,7 @@ public class HDFSRepositoryConnector ext
{
String nameNodeProtocol = parameters.getParameter("namenodeprotocol");
if (nameNodeProtocol == null) {
- nameNodeProtocol = "file";
+ nameNodeProtocol = "hdfs";
}
String nameNodeHost = parameters.getParameter("namenodehost");
@@ -633,6 +633,7 @@ public class HDFSRepositoryConnector ext
" <option value=\"file\"" + (nameNodeProtocol.equals("file")?" selected=\"true\"":"") + ">file</option>\n"+
" <option value=\"ftp\"" + (nameNodeProtocol.equals("ftp")?" selected=\"true\"":"") + ">ftp</option>\n"+
" <option value=\"har\"" + (nameNodeProtocol.equals("har")?" selected=\"true\"":"") + ">har</option>\n"+
+" <option value=\"hdfs\"" + (nameNodeProtocol.equals("hdfs")?" selected=\"true\"":"") + ">hdfs</option>\n"+
" <option value=\"s3\"" + (nameNodeProtocol.equals("s3")?" selected=\"true\"":"") + ">s3</option>\n"+
" <option value=\"s3n\"" + (nameNodeProtocol.equals("s3n")?" selected=\"true\"":"") + ">s3n</option>\n"+
" <option value=\"viewfs\"" + (nameNodeProtocol.equals("viewfs")?" selected=\"true\"":"") + ">viewfs</option>\n"+
@@ -721,7 +722,7 @@ public class HDFSRepositoryConnector ext
{
String nameNodeProtocol = parameters.getParameter("namenodeprotocol");
if (nameNodeProtocol == null)
- nameNodeProtocol = "file";
+ nameNodeProtocol = "hdfs";
String nameNodeHost = parameters.getParameter("namenodehost");
String nameNodePort = parameters.getParameter("namenodeport");
Modified: manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/editConfiguration.html
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/editConfiguration.html?rev=1559005&r1=1559004&r2=1559005&view=diff
==============================================================================
--- manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/editConfiguration.html (original)
+++ manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/editConfiguration.html Fri Jan 17 03:27:36 2014
@@ -38,6 +38,11 @@
#else
<option value="har">har</option>
#end
+ #if($NAMENODEPROTOCOL == 'hdfs')
+ <option value="hdfs" selected="true">hdfs</option>
+ #else
+ <option value="hdfs">hdfs</option>
+ #end
#if($NAMENODEPROTOCOL == 's3')
<option value="s3" selected="true">s3</option>
#else