You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by mi...@apache.org on 2013/06/24 05:26:58 UTC
svn commit: r1495913 [1/5] - in /manifoldcf/branches/CONNECTORS-728: ./
connectors/
connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/
connectors/filesystem/connector/src/main/native2ascii/org/apache/mani...
Author: minoru
Date: Mon Jun 24 03:26:56 2013
New Revision: 1495913
URL: http://svn.apache.org/r1495913
Log:
Add HDFS Connector.
Added:
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/build.xml (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConfig.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConstant.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputParam.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputSpecs.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/Messages.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/Messages.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_en_US.properties (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_ja_JP.properties (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/hdfs/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/hdfs/common_en_US.properties (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/hdfs/common_ja_JP.properties (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/editConfiguration.html (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/editConfiguration.js (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/editSpecification.html (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/editSpecification.js (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/viewConfiguration.html (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/viewSpecification.html (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/hdfs/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/hdfs/BaseDerby.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/hdfs/BaseHSQLDB.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/hdfs/BaseMySQL.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/hdfs/BasePostgresql.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/hdfs/SanityDerbyTest.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/hdfs/SanityHSQLDBTest.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/hdfs/SanityMySQLTest.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/hdfs/SanityPostgresqlTest.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/tests/
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/tests/BaseDerby.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/tests/BaseHSQLDB.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/tests/BaseMySQL.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/tests/BasePostgresql.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/tests/SanityDerbyTest.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/tests/SanityHSQLDBTest.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/tests/SanityMySQLTest.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/tests/SanityPostgresqlTest.java (with props)
manifoldcf/branches/CONNECTORS-728/connectors/hdfs/pom.xml (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/
manifoldcf/branches/CONNECTORS-728/tests/hdfs/build.xml (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/pom.xml (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityCombinedHSQLDBIT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityDerbyIT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityHSQLDBIT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityMySQLIT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityPostgresqlIT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityTester.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseDerby.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseHSQLDB.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseMySQL.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BasePostgresql.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseUIDerby.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseUIHSQLDB.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlDerbyLT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlHSQLDBLT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlMySQLLT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlPostgresqlLT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlTester.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationDerbyIT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationHSQLDBIT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationMySQLIT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationPostgresqlIT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationTester.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/FileHelper.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountDerbyIT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountHSQLDBIT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountMySQLIT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountPostgresqlIT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountTester.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/NavigationCombinedHSQLDBUI.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/NavigationDerbyUI.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/NavigationUITester.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/SanityDerbyIT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/SanityHSQLDBIT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/SanityMySQLIT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/SanityPostgresqlIT.java (with props)
manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/SanityTester.java (with props)
Modified:
manifoldcf/branches/CONNECTORS-728/build.xml
manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java
manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_en_US.properties
manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_ja_JP.properties
manifoldcf/branches/CONNECTORS-728/connectors/pom.xml
Modified: manifoldcf/branches/CONNECTORS-728/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/build.xml?rev=1495913&r1=1495912&r2=1495913&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/build.xml (original)
+++ manifoldcf/branches/CONNECTORS-728/build.xml Mon Jun 24 03:26:56 2013
@@ -66,6 +66,7 @@
<ant dir="connectors/filenet" target="clean"/>
<ant dir="connectors/filesystem" target="clean"/>
<ant dir="connectors/gts" target="clean"/>
+ <ant dir="connectors/hdfs" target="clean"/>
<ant dir="connectors/jcifs" target="clean"/>
<ant dir="connectors/jdbc" target="clean"/>
<ant dir="connectors/livelink" target="clean"/>
@@ -86,6 +87,7 @@
<ant dir="tests/cmis" target="clean"/>
<ant dir="tests/filesystem" target="clean"/>
<ant dir="tests/gts" target="clean"/>
+ <ant dir="tests/hdfs" target="clean"/>
<ant dir="tests/opensearchserver" target="clean"/>
<ant dir="tests/rss" target="clean"/>
<ant dir="tests/solr" target="clean"/>
@@ -118,6 +120,7 @@
<ant dir="connectors/filenet" target="clean"/>
<ant dir="connectors/filesystem" target="clean"/>
<ant dir="connectors/gts" target="clean"/>
+ <ant dir="connectors/hdfs" target="clean"/>
<ant dir="connectors/jcifs" target="clean"/>
<ant dir="connectors/jdbc" target="clean"/>
<ant dir="connectors/livelink" target="clean"/>
@@ -138,6 +141,7 @@
<ant dir="tests/cmis" target="clean"/>
<ant dir="tests/filesystem" target="clean"/>
<ant dir="tests/gts" target="clean"/>
+ <ant dir="tests/hdfs" target="clean"/>
<ant dir="tests/opensearchserver" target="clean"/>
<ant dir="tests/rss" target="clean"/>
<ant dir="tests/solr" target="clean"/>
@@ -521,6 +525,42 @@
<ant dir="connectors/gts" target="run-tests-HSQLDB"/>
</target>
+ <target name="setup-hdfs-connector" depends="build-framework" if="downloaded"/>
+
+ <target name="setup-hdfs-connector-tests" depends="build-tests-framework" if="downloaded"/>
+
+ <target name="build-hdfs-connector" depends="setup-hdfs-connector" if="downloaded">
+ <ant dir="connectors/hdfs" target="build"/>
+ </target>
+
+ <target name="doc-hdfs-connector" depends="setup-hdfs-connector" if="downloaded">
+ <ant dir="connectors/hdfs" target="doc"/>
+ </target>
+
+ <target name="build-tests-hdfs-connector" depends="setup-hdfs-connector,setup-hdfs-connector-tests" if="downloaded">
+ <ant dir="connectors/hdfs" target="build-tests"/>
+ </target>
+
+ <target name="run-tests-hdfs-connector" depends="setup-hdfs-connector,setup-hdfs-connector-tests" if="downloaded">
+ <ant dir="connectors/hdfs" target="run-tests"/>
+ </target>
+
+ <target name="run-tests-derby-hdfs-connector" depends="setup-hdfs-connector,setup-hdfs-connector-tests" if="downloaded">
+ <ant dir="connectors/hdfs" target="run-tests-derby"/>
+ </target>
+
+ <target name="run-tests-postgresql-hdfs-connector" depends="setup-hdfs-connector,setup-hdfs-connector-tests" if="downloaded">
+ <ant dir="connectors/hdfs" target="run-tests-postgresql"/>
+ </target>
+
+ <target name="run-tests-mysql-hdfs-connector" depends="setup-hdfs-connector,setup-hdfs-connector-tests" if="downloaded">
+ <ant dir="connectors/hdfs" target="run-tests-mysql"/>
+ </target>
+
+ <target name="run-tests-HSQLDB-hdfs-connector" depends="setup-hdfs-connector,setup-hdfs-connector-tests" if="downloaded">
+ <ant dir="connectors/hdfs" target="run-tests-HSQLDB"/>
+ </target>
+
<target name="setup-jcifs-connector" depends="build-framework" if="downloaded"/>
<target name="setup-jcifs-connector-tests" depends="build-tests-framework" if="downloaded"/>
@@ -1668,6 +1708,48 @@
</condition>
</target>
+ <target name="calculate-hdfs-condition" depends="build-hdfs-connector">
+ <available file="connectors/hdfs/dist/lib" type="dir" property="hdfs.exists"/>
+ <condition property="hdfs.include">
+ <and>
+ <isset property="hdfs.exists"/>
+ <isset property="downloaded"/>
+ </and>
+ </condition>
+ </target>
+
+ <target name="calculate-hdfs-doc-condition" depends="doc-hdfs-connector">
+ <available file="connectors/hdfs/dist/doc" type="dir" property="hdfs-doc.exists"/>
+ <condition property="hdfs-doc.include">
+ <and>
+ <isset property="hdfs-doc.exists"/>
+ <isset property="downloaded"/>
+ </and>
+ </condition>
+ </target>
+
+ <target name="deliver-hdfs-connector" depends="calculate-hdfs-condition" if="hdfs.include">
+ <antcall target="general-connector-delivery">
+ <param name="connector-name" value="hdfs"/>
+ </antcall>
+ <antcall target="general-add-repository-connector">
+ <param name="connector-name" value="hdfs"/>
+ <param name="connector-label" value="HDFS"/>
+ <param name="connector-class" value="org.apache.manifoldcf.crawler.connectors.hdfs.HDFSRepositoryConnector"/>
+ </antcall>
+ <antcall target="general-add-output-connector">
+ <param name="connector-name" value="hdfs"/>
+ <param name="connector-label" value="HDFS"/>
+ <param name="connector-class" value="org.apache.manifoldcf.agents.output.hdfs.HDFSOutputConnector"/>
+ </antcall>
+ </target>
+
+ <target name="deliver-hdfs-connector-doc" depends="calculate-hdfs-doc-condition" if="hdfs-doc.include">
+ <antcall target="general-connector-doc-delivery">
+ <param name="connector-name" value="hdfs"/>
+ </antcall>
+ </target>
+
<target name="calculate-jdbc-doc-condition" depends="doc-jdbc-connector">
<available file="connectors/jdbc/dist/doc" type="dir" property="jdbc-doc.exists"/>
<condition property="jdbc-doc.include">
@@ -2223,7 +2305,16 @@
</condition>
</target>
- <target name="calculate-jcifs-tests-condition" depends="calculate-jcifs-condition,calculate-nulloutput-condition">
+ <target name="calculate-hdfs-tests-condition" depends="calculate-hdfs-condition,calculate-nulloutput-condition">
+ <condition property="hdfs-tests.include">
+ <and>
+ <isset property="hdfs.include"/>
+ <isset property="nulloutput.include"/>
+ </and>
+ </condition>
+ </target>
+
+ <target name="calculate-jcifs-tests-condition" depends="calculate-jcifs-condition,calculate-nulloutput-condition">
<condition property="jcifs-tests.include">
<and>
<isset property="jcifs.include"/>
@@ -2358,6 +2449,18 @@
<ant dir="tests/filesystem" target="run-load-derby"/>
</target>
+ <target name="run-hdfs-tests-derby" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+ <ant dir="tests/hdfs" target="run-derby"/>
+ </target>
+
+ <target name="run-hdfs-UI-tests-derby" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+ <ant dir="tests/hdfs" target="run-UI-derby"/>
+ </target>
+
+ <target name="run-hdfs-loadtests-derby" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+ <ant dir="tests/hdfs" target="run-load-derby"/>
+ </target>
+
<target name="run-jcifs-UI-tests-derby" depends="build-tests-framework,build-tests-jcifs-connector,build-tests-nulloutput-connector,calculate-jcifs-tests-condition" if="jcifs-tests.include">
<ant dir="tests/jcifs" target="run-UI-derby"/>
</target>
@@ -2502,6 +2605,22 @@
<ant dir="tests/filesystem" target="run-load-mysql"/>
</target>
+ <target name="run-hdfs-tests-postgresql" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+ <ant dir="tests/hdfs" target="run-postgresql"/>
+ </target>
+
+ <target name="run-hdfs-tests-mysql" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+ <ant dir="tests/hdfs" target="run-mysql"/>
+ </target>
+
+ <target name="run-hdfs-loadtests-postgresql" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+ <ant dir="tests/hdfs" target="run-load-postgresql"/>
+ </target>
+
+ <target name="run-hdfs-loadtests-mysql" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+ <ant dir="tests/hdfs" target="run-load-mysql"/>
+ </target>
+
<target name="run-wiki-tests-postgresql" depends="build-tests-framework,build-tests-wiki-connector,build-tests-nulloutput-connector,calculate-wiki-tests-condition" if="wiki-tests.include">
<ant dir="tests/wiki" target="run-postgresql"/>
</target>
@@ -2566,6 +2685,18 @@
<ant dir="tests/filesystem" target="run-load-HSQLDB"/>
</target>
+ <target name="run-hdfs-tests-HSQLDB" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+ <ant dir="tests/hdfs" target="run-HSQLDB"/>
+ </target>
+
+ <target name="run-hdfs-UI-tests-HSQLDB" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+ <ant dir="tests/hdfs" target="run-UI-HSQLDB"/>
+ </target>
+
+ <target name="run-hdfs-loadtests-HSQLDB" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+ <ant dir="tests/hdfs" target="run-load-HSQLDB"/>
+ </target>
+
<target name="run-wiki-tests-HSQLDB" depends="build-tests-framework,build-tests-wiki-connector,build-tests-nulloutput-connector,calculate-wiki-tests-condition" if="wiki-tests.include">
<ant dir="tests/wiki" target="run-HSQLDB"/>
</target>
@@ -2614,45 +2745,45 @@
<ant dir="tests/sharepoint" target="run-load-HSQLDB"/>
</target>
- <target name="run-tests-open-connectors" depends="run-tests-activedirectory-connector,run-tests-ldap-connector,run-tests-alfresco-connector,run-tests-cmis-connector,run-tests-filesystem-connector,run-tests-nullauthority-connector,run-tests-nulloutput-connector,run-tests-rss-connector,run-tests-solr-connector,run-tests-webcrawler-connector,run-tests-wiki-connector,run-tests-jdbc-connector"/>
+ <target name="run-tests-open-connectors" depends="run-tests-activedirectory-connector,run-tests-ldap-connector,run-tests-alfresco-connector,run-tests-cmis-connector,run-tests-filesystem-connector,run-tests-hdfs-connector,run-tests-nullauthority-connector,run-tests-nulloutput-connector,run-tests-rss-connector,run-tests-solr-connector,run-tests-webcrawler-connector,run-tests-wiki-connector,run-tests-jdbc-connector"/>
<target name="run-tests-lgpl-connectors" depends="run-tests-jcifs-connector"/>
<target name="run-tests-proprietary-connectors" depends="run-tests-documentum-connector,run-tests-filenet-connector,run-tests-livelink-connector,run-tests-memex-connector,run-tests-meridio-connector,run-tests-sharepoint-connector"/>
- <target name="run-tests-derby-open-connectors" depends="run-tests-derby-activedirectory-connector,run-tests-derby-ldap-connector,run-tests-derby-alfresco-connector,run-tests-derby-cmis-connector,run-tests-derby-filesystem-connector,run-tests-derby-nullauthority-connector,run-tests-derby-nulloutput-connector,run-tests-derby-rss-connector,run-tests-derby-solr-connector,run-tests-derby-webcrawler-connector,run-tests-derby-wiki-connector,run-tests-derby-jdbc-connector"/>
+ <target name="run-tests-derby-open-connectors" depends="run-tests-derby-activedirectory-connector,run-tests-derby-ldap-connector,run-tests-derby-alfresco-connector,run-tests-derby-cmis-connector,run-tests-derby-filesystem-connector,run-tests-derby-hdfs-connector,run-tests-derby-nullauthority-connector,run-tests-derby-nulloutput-connector,run-tests-derby-rss-connector,run-tests-derby-solr-connector,run-tests-derby-webcrawler-connector,run-tests-derby-wiki-connector,run-tests-derby-jdbc-connector"/>
<target name="run-tests-derby-lgpl-connectors" depends="run-tests-derby-jcifs-connector"/>
<target name="run-tests-derby-proprietary-connectors" depends="run-tests-derby-documentum-connector,run-tests-derby-filenet-connector,run-tests-derby-livelink-connector,run-tests-derby-memex-connector,run-tests-derby-meridio-connector,run-tests-derby-sharepoint-connector"/>
- <target name="end-to-end-tests-derby" depends="run-filesystem-tests-derby,run-webcrawler-tests-derby,run-rss-tests-derby,run-solr-tests-derby,run-wiki-tests-derby,run-alfresco-tests-derby,run-cmis-tests-derby,run-sharepoint-tests-derby"/>
+ <target name="end-to-end-tests-derby" depends="run-filesystem-tests-derby,run-hdfs-tests-derby,run-webcrawler-tests-derby,run-rss-tests-derby,run-solr-tests-derby,run-wiki-tests-derby,run-alfresco-tests-derby,run-cmis-tests-derby,run-sharepoint-tests-derby"/>
- <target name="run-tests-postgresql-open-connectors" depends="run-tests-postgresql-activedirectory-connector,run-tests-postgresql-ldap-connector,run-tests-postgresql-alfresco-connector,run-tests-postgresql-cmis-connector,run-tests-postgresql-filesystem-connector,run-tests-postgresql-nullauthority-connector,run-tests-postgresql-nulloutput-connector,run-tests-postgresql-rss-connector,run-tests-postgresql-solr-connector,run-tests-postgresql-webcrawler-connector,run-tests-postgresql-wiki-connector,run-tests-postgresql-jdbc-connector,run-tests-postgresql-opensearchserver-connector,run-tests-postgresql-elasticsearch-connector"/>
+ <target name="run-tests-postgresql-open-connectors" depends="run-tests-postgresql-activedirectory-connector,run-tests-postgresql-ldap-connector,run-tests-postgresql-alfresco-connector,run-tests-postgresql-cmis-connector,run-tests-postgresql-filesystem-connector,run-tests-postgresql-hdfs-connector,run-tests-postgresql-nullauthority-connector,run-tests-postgresql-nulloutput-connector,run-tests-postgresql-rss-connector,run-tests-postgresql-solr-connector,run-tests-postgresql-webcrawler-connector,run-tests-postgresql-wiki-connector,run-tests-postgresql-jdbc-connector,run-tests-postgresql-opensearchserver-connector,run-tests-postgresql-elasticsearch-connector"/>
<target name="run-tests-postgresql-lgpl-connectors" depends="run-tests-postgresql-jcifs-connector"/>
<target name="run-tests-postgresql-proprietary-connectors" depends="run-tests-postgresql-documentum-connector,run-tests-postgresql-filenet-connector,run-tests-postgresql-livelink-connector,run-tests-postgresql-memex-connector,run-tests-postgresql-meridio-connector,run-tests-postgresql-sharepoint-connector"/>
- <target name="end-to-end-tests-postgresql" depends="run-filesystem-tests-postgresql,run-webcrawler-tests-postgresql,run-wiki-tests-postgresql,run-alfresco-tests-postgresql,run-cmis-tests-postgresql,run-sharepoint-tests-postgresql"/>
+ <target name="end-to-end-tests-postgresql" depends="run-filesystem-tests-postgresql,run-hdfs-tests-postgresql,run-webcrawler-tests-postgresql,run-wiki-tests-postgresql,run-alfresco-tests-postgresql,run-cmis-tests-postgresql,run-sharepoint-tests-postgresql"/>
- <target name="run-tests-mysql-open-connectors" depends="run-tests-mysql-activedirectory-connector,run-tests-mysql-ldap-connector,run-tests-mysql-alfresco-connector,run-tests-mysql-cmis-connector,run-tests-mysql-filesystem-connector,run-tests-mysql-nullauthority-connector,run-tests-mysql-nulloutput-connector,run-tests-mysql-rss-connector,run-tests-mysql-solr-connector,run-tests-mysql-webcrawler-connector,run-tests-mysql-wiki-connector,run-tests-mysql-jdbc-connector,run-tests-mysql-opensearchserver-connector,run-tests-mysql-elasticsearch-connector"/>
+ <target name="run-tests-mysql-open-connectors" depends="run-tests-mysql-activedirectory-connector,run-tests-mysql-ldap-connector,run-tests-mysql-alfresco-connector,run-tests-mysql-cmis-connector,run-tests-mysql-filesystem-connector,run-tests-mysql-hdfs-connector,run-tests-mysql-nullauthority-connector,run-tests-mysql-nulloutput-connector,run-tests-mysql-rss-connector,run-tests-mysql-solr-connector,run-tests-mysql-webcrawler-connector,run-tests-mysql-wiki-connector,run-tests-mysql-jdbc-connector,run-tests-mysql-opensearchserver-connector,run-tests-mysql-elasticsearch-connector"/>
<target name="run-tests-mysql-lgpl-connectors" depends="run-tests-mysql-jcifs-connector"/>
<target name="run-tests-mysql-proprietary-connectors" depends="run-tests-mysql-documentum-connector,run-tests-mysql-filenet-connector,run-tests-mysql-livelink-connector,run-tests-mysql-memex-connector,run-tests-mysql-meridio-connector,run-tests-mysql-sharepoint-connector"/>
- <target name="end-to-end-tests-mysql" depends="run-filesystem-tests-mysql,run-webcrawler-tests-mysql,run-wiki-tests-mysql,run-alfresco-tests-mysql,run-cmis-tests-mysql,run-sharepoint-tests-mysql"/>
+ <target name="end-to-end-tests-mysql" depends="run-filesystem-tests-mysql,run-hdfs-tests-mysql,run-webcrawler-tests-mysql,run-wiki-tests-mysql,run-alfresco-tests-mysql,run-cmis-tests-mysql,run-sharepoint-tests-mysql"/>
- <target name="run-tests-HSQLDB-open-connectors" depends="run-tests-HSQLDB-activedirectory-connector,run-tests-HSQLDB-ldap-connector,run-tests-HSQLDB-alfresco-connector,run-tests-HSQLDB-cmis-connector,run-tests-HSQLDB-filesystem-connector,run-tests-HSQLDB-nullauthority-connector,run-tests-HSQLDB-nulloutput-connector,run-tests-HSQLDB-rss-connector,run-tests-HSQLDB-solr-connector,run-tests-HSQLDB-webcrawler-connector,run-tests-HSQLDB-wiki-connector,run-tests-HSQLDB-jdbc-connector,run-tests-HSQLDB-opensearchserver-connector,run-tests-HSQLDB-elasticsearch-connector"/>
+ <target name="run-tests-HSQLDB-open-connectors" depends="run-tests-HSQLDB-activedirectory-connector,run-tests-HSQLDB-ldap-connector,run-tests-HSQLDB-alfresco-connector,run-tests-HSQLDB-cmis-connector,run-tests-HSQLDB-filesystem-connector,run-tests-HSQLDB-hdfs-connector,run-tests-HSQLDB-nullauthority-connector,run-tests-HSQLDB-nulloutput-connector,run-tests-HSQLDB-rss-connector,run-tests-HSQLDB-solr-connector,run-tests-HSQLDB-webcrawler-connector,run-tests-HSQLDB-wiki-connector,run-tests-HSQLDB-jdbc-connector,run-tests-HSQLDB-opensearchserver-connector,run-tests-HSQLDB-elasticsearch-connector"/>
<target name="run-tests-HSQLDB-lgpl-connectors" depends="run-tests-HSQLDB-jcifs-connector"/>
<target name="run-tests-HSQLDB-proprietary-connectors" depends="run-tests-HSQLDB-documentum-connector,run-tests-HSQLDB-filenet-connector,run-tests-HSQLDB-livelink-connector,run-tests-HSQLDB-memex-connector,run-tests-HSQLDB-meridio-connector,run-tests-HSQLDB-sharepoint-connector"/>
- <target name="end-to-end-tests-HSQLDB" depends="run-filesystem-tests-HSQLDB,run-rss-tests-HSQLDB,run-wiki-tests-HSQLDB,run-alfresco-tests-HSQLDB,run-cmis-tests-HSQLDB,run-sharepoint-tests-HSQLDB"/>
+ <target name="end-to-end-tests-HSQLDB" depends="run-filesystem-tests-HSQLDB,run-hdfs-tests-HSQLDB,run-rss-tests-HSQLDB,run-wiki-tests-HSQLDB,run-alfresco-tests-HSQLDB,run-cmis-tests-HSQLDB,run-sharepoint-tests-HSQLDB"/>
- <target name="end-to-end-loadtests-derby" depends="run-filesystem-loadtests-derby,run-rss-loadtests-derby,run-wiki-loadtests-derby,run-alfresco-loadtests-derby,run-cmis-loadtests-derby,run-sharepoint-loadtests-derby"/>
+ <target name="end-to-end-loadtests-derby" depends="run-filesystem-loadtests-derby,run-hdfs-loadtests-derby,run-rss-loadtests-derby,run-wiki-loadtests-derby,run-alfresco-loadtests-derby,run-cmis-loadtests-derby,run-sharepoint-loadtests-derby"/>
- <target name="end-to-end-loadtests-postgresql" depends="run-filesystem-loadtests-postgresql,run-rss-loadtests-postgresql,run-wiki-loadtests-postgresql,run-alfresco-loadtests-postgresql,run-cmis-loadtests-postgresql,run-sharepoint-loadtests-postgresql"/>
+ <target name="end-to-end-loadtests-postgresql" depends="run-filesystem-loadtests-postgresql,run-hdfs-loadtests-postgresql,run-rss-loadtests-postgresql,run-wiki-loadtests-postgresql,run-alfresco-loadtests-postgresql,run-cmis-loadtests-postgresql,run-sharepoint-loadtests-postgresql"/>
- <target name="end-to-end-loadtests-mysql" depends="run-filesystem-loadtests-mysql,run-rss-loadtests-mysql,run-wiki-loadtests-mysql,run-alfresco-loadtests-mysql,run-cmis-loadtests-mysql,run-sharepoint-loadtests-mysql"/>
+ <target name="end-to-end-loadtests-mysql" depends="run-filesystem-loadtests-mysql,run-hdfs-loadtests-mysql,run-rss-loadtests-mysql,run-wiki-loadtests-mysql,run-alfresco-loadtests-mysql,run-cmis-loadtests-mysql,run-sharepoint-loadtests-mysql"/>
- <target name="end-to-end-loadtests-HSQLDB" depends="run-filesystem-loadtests-HSQLDB,run-rss-loadtests-HSQLDB,run-wiki-loadtests-HSQLDB,run-alfresco-loadtests-HSQLDB,run-cmis-loadtests-HSQLDB,run-sharepoint-loadtests-HSQLDB"/>
+ <target name="end-to-end-loadtests-HSQLDB" depends="run-filesystem-loadtests-HSQLDB,run-hdfs-loadtests-HSQLDB,run-rss-loadtests-HSQLDB,run-wiki-loadtests-HSQLDB,run-alfresco-loadtests-HSQLDB,run-cmis-loadtests-HSQLDB,run-sharepoint-loadtests-HSQLDB"/>
- <target name="deliver-open-connectors" depends="deliver-googledrive-connector,deliver-dropbox-connector,deliver-nullauthority-connector,deliver-activedirectory-connector,deliver-ldap-connector,deliver-alfresco-connector,deliver-cmis-connector,deliver-filesystem-connector,deliver-rss-connector,deliver-webcrawler-connector,deliver-wiki-connector,deliver-jdbc-connector"/>
- <target name="deliver-open-connectors-doc" depends="deliver-googledrive-connector-doc,deliver-dropbox-connector-doc,deliver-nullauthority-connector-doc,deliver-activedirectory-connector-doc,deliver-ldap-connector-doc,deliver-alfresco-connector-doc,deliver-cmis-connector-doc,deliver-filesystem-connector-doc,deliver-rss-connector-doc,deliver-webcrawler-connector-doc,deliver-wiki-connector-doc,deliver-jdbc-connector-doc"/>
+ <target name="deliver-open-connectors" depends="deliver-googledrive-connector,deliver-dropbox-connector,deliver-nullauthority-connector,deliver-activedirectory-connector,deliver-ldap-connector,deliver-alfresco-connector,deliver-cmis-connector,deliver-filesystem-connector,deliver-hdfs-connector,deliver-rss-connector,deliver-webcrawler-connector,deliver-wiki-connector,deliver-jdbc-connector"/>
+ <target name="deliver-open-connectors-doc" depends="deliver-googledrive-connector-doc,deliver-dropbox-connector-doc,deliver-nullauthority-connector-doc,deliver-activedirectory-connector-doc,deliver-ldap-connector-doc,deliver-alfresco-connector-doc,deliver-cmis-connector-doc,deliver-filesystem-connector-doc,deliver-hdfs-connector-doc,deliver-rss-connector-doc,deliver-webcrawler-connector-doc,deliver-wiki-connector-doc,deliver-jdbc-connector-doc"/>
<target name="deliver-output-connectors" depends="deliver-gts-connector,deliver-solr-connector,deliver-nulloutput-connector,deliver-opensearchserver-connector,deliver-elasticsearch-connector"/>
<target name="deliver-output-connectors-doc" depends="deliver-gts-connector-doc,deliver-solr-connector-doc,deliver-nulloutput-connector-doc,deliver-opensearchserver-connector-doc,deliver-elasticsearch-connector-doc"/>
@@ -2803,7 +2934,7 @@
<target name="ldtest" depends="load-dr,load-hs"/>
- <target name="uitest" depends="run-filesystem-UI-tests-derby,run-filesystem-UI-tests-HSQLDB,run-jcifs-UI-tests-derby,run-jdbc-UI-tests-derby,run-activedirectory-UI-tests-derby,run-ldap-UI-tests-derby,run-rss-UI-tests-derby,run-webcrawler-UI-tests-derby,run-wiki-UI-tests-derby,run-solr-UI-tests-derby,run-cmis-UI-tests-derby,run-gts-UI-tests-derby,run-opensearchserver-UI-tests-derby"/>
+ <target name="uitest" depends="run-filesystem-UI-tests-derby,run-hdfs-UI-tests-derby,run-filesystem-UI-tests-HSQLDB,run-hdfs-UI-tests-HSQLDB,run-jcifs-UI-tests-derby,run-jdbc-UI-tests-derby,run-activedirectory-UI-tests-derby,run-ldap-UI-tests-derby,run-rss-UI-tests-derby,run-webcrawler-UI-tests-derby,run-wiki-UI-tests-derby,run-solr-UI-tests-derby,run-cmis-UI-tests-derby,run-gts-UI-tests-derby,run-opensearchserver-UI-tests-derby"/>
<target name="all" depends="build,javadoc,doc,image,test-dr,test-hs"/>
@@ -3200,6 +3331,12 @@ Use Apache Forrest version forrest-0.9-d
<param name="artifact-name" value="commons-logging"/>
<param name="artifact-type" value="jar"/>
</antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="commons-configuration"/>
+ <param name="artifact-version" value="1.6"/>
+ <param name="artifact-name" value="commons-configuration"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
</target>
<target name="download-slf4j">
@@ -3743,7 +3880,18 @@ Use Apache Forrest version forrest-0.9-d
</copy>
</target>
- <target name="download-solrj">
+ <target name="download-hadoop">
+ <mkdir dir="lib"/>
+ <antcall target="download-via-maven">
+ <param name="target" value="lib"/>
+ <param name="project-path" value="org/apache/hadoop"/>
+ <param name="artifact-version" value="1.0.4"/>
+ <param name="artifact-name" value="hadoop-core"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ </target>
+
+ <target name="download-solrj">
<mkdir dir="lib"/>
<antcall target="download-via-maven">
<param name="target" value="lib"/>
@@ -3761,7 +3909,7 @@ Use Apache Forrest version forrest-0.9-d
</antcall>
</target>
- <target name="make-core-deps" depends="download-google-api-client,download-dropbox-client,download-solrj,download-httpcomponents,download-json,download-hsqldb,download-xerces,download-commons,download-elasticsearch-plugin,download-solr-plugins,download-sharepoint-plugins,download-jstl,download-xmlgraphics-commons,download-wstx-asl,download-xmlsec,download-xml-apis,download-wss4j,download-velocity,download-streambuffer,download-stax,download-servlet-api,download-xml-resolver,download-osgi,download-opensaml,download-mimepull,download-mail,download-log4j,download-junit,download-jaxws,download-glassfish,download-jaxb,download-tomcat,download-h2,download-h2-support,download-geronimo-specs,download-fop,download-derby,download-postgresql,download-axis,download-saaj,download-wsdl4j,download-castor,download-jetty,download-slf4j,download-xalan,download-activation,download-avalon-framework,download-poi,download-chemistry,download-ecj">
+ <target name="make-core-deps" depends="download-google-api-client,download-dropbox-client,download-solrj,download-httpcomponents,download-json,download-hsqldb,download-xerces,download-commons,download-elasticsearch-plugin,download-solr-plugins,download-sharepoint-plugins,download-jstl,download-xmlgraphics-commons,download-wstx-asl,download-xmlsec,download-xml-apis,download-wss4j,download-velocity,download-streambuffer,download-stax,download-servlet-api,download-xml-resolver,download-osgi,download-opensaml,download-mimepull,download-mail,download-log4j,download-junit,download-jaxws,download-glassfish,download-jaxb,download-tomcat,download-h2,download-h2-support,download-geronimo-specs,download-fop,download-derby,download-postgresql,download-axis,download-saaj,download-wsdl4j,download-castor,download-jetty,download-slf4j,download-xalan,download-activation,download-avalon-framework,download-poi,download-chemistry,download-ecj,download-hadoop">
<copy todir="lib">
<fileset dir="lib-license" includes="*.txt"/>
</copy>
@@ -3798,6 +3946,7 @@ Use Apache Forrest version forrest-0.9-d
<ant dir="connectors/filenet" target="download-dependencies"/>
<ant dir="connectors/filesystem" target="download-dependencies"/>
<ant dir="connectors/gts" target="download-dependencies"/>
+ <ant dir="connectors/hdfs" target="download-dependencies"/>
<ant dir="connectors/jcifs" target="download-dependencies"/>
<ant dir="connectors/jdbc" target="download-dependencies"/>
<ant dir="connectors/livelink" target="download-dependencies"/>
@@ -3836,6 +3985,7 @@ Use Apache Forrest version forrest-0.9-d
<ant dir="connectors/filenet" target="download-cleanup"/>
<ant dir="connectors/filesystem" target="download-cleanup"/>
<ant dir="connectors/gts" target="download-cleanup"/>
+ <ant dir="connectors/hdfs" target="download-cleanup"/>
<ant dir="connectors/jcifs" target="download-cleanup"/>
<ant dir="connectors/jdbc" target="download-cleanup"/>
<ant dir="connectors/livelink" target="download-cleanup"/>
Modified: manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java?rev=1495913&r1=1495912&r2=1495913&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java Mon Jun 24 03:26:56 2013
@@ -821,7 +821,7 @@ public class FileConnector extends org.a
" <tr>\n"+
" <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"FileConnector.FilePathToURI") + "</nobr></td>\n"+
" <td class=\"value\">\n"+
-" <input name=\"filepathtouri\" type=\"checkbox\" value=\"true\"" + (filePathToUri ? "checked" : "") +"/>\n" +
+" <input name=\"filepathtouri\" type=\"checkbox\" value=\"true\"" + (filePathToUri ? "checked" : "") +"/> " + Messages.getBodyString(locale,"FileConnector.FilePathToURIExample") + "\n" +
" </td>\n"+
" </tr>\n"+
"</table>\n"
Modified: manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_en_US.properties?rev=1495913&r1=1495912&r2=1495913&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_en_US.properties (original)
+++ manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_en_US.properties Mon Jun 24 03:26:56 2013
@@ -38,3 +38,4 @@ FileConnector.AddNewPath=Add new path
FileConnector.FilePathToURITab=Convert file path to URI
FileConnector.FilePathToURI=Convert file path to URI:
+FileConnector.FilePathToURIExample=EX) $REPOSITORY_PATH/http/localhost/index.html => http://localhost/index.html
Modified: manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_ja_JP.properties?rev=1495913&r1=1495912&r2=1495913&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_ja_JP.properties (original)
+++ manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_ja_JP.properties Mon Jun 24 03:26:56 2013
@@ -38,3 +38,4 @@ FileConnector.AddNewPath=æ°ãã�
FileConnector.FilePathToURITab=ãã¡ã¤ã«ãã¹ãURIã¸å¤æãã
FileConnector.FilePathToURI=ãã¡ã¤ã«ãã¹ãURIã¸å¤æããï¼
+FileConnector.FilePathToURIExample=ä¾) $REPOSITORY_PATH/http/localhost/index.html => http://localhost/index.html
Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/build.xml?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/build.xml (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/build.xml Mon Jun 24 03:26:56 2013
@@ -0,0 +1,39 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project name="hdfs" default="all">
+
+ <import file="../connector-build.xml"/>
+
+ <path id="connector-classpath">
+ <path refid="mcf-connector-build.connector-classpath"/>
+ <fileset dir="../../lib">
+ <include name="commons-configuration*.jar"/>
+ <include name="hadoop-core*.jar"/>
+ </fileset>
+ </path>
+
+ <target name="lib" depends="mcf-connector-build.lib,precompile-check" if="canBuild">
+ <mkdir dir="dist/lib"/>
+ <copy todir="dist/lib">
+ <fileset dir="../../lib">
+ <include name="commons-configuration*.jar"/>
+ <include name="hadoop-core*.jar"/>
+ </fileset>
+ </copy>
+ </target>
+</project>
Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/build.xml
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConfig.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConfig.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConfig.java (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConfig.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,71 @@
+/* $Id: FileOutputConfig.java 1299512 2013-05-31 22:59:38Z minoru $ */
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.manifoldcf.agents.output.hdfs;
+
+import org.apache.manifoldcf.core.interfaces.ConfigParams;
+import org.apache.manifoldcf.core.interfaces.IPostParameters;
+
+
+public class HDFSOutputConfig extends HDFSOutputParam {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = -2062295503498352538L;
+
+ /** Parameters used for the configuration */
+ final private static ParameterEnum[] CONFIGURATIONLIST = {
+ ParameterEnum.NAMENODE,
+ ParameterEnum.USER
+ };
+
+ /** Build a set of ElasticSearchParameters by reading ConfigParams. If the
+ * value returned by ConfigParams.getParameter is null, the default value is
+ * set.
+ *
+ * @param paramList
+ * @param params
+ */
+ public HDFSOutputConfig(ConfigParams params)
+ {
+ super(CONFIGURATIONLIST);
+ for (ParameterEnum param : CONFIGURATIONLIST) {
+ String value = params.getParameter(param.name());
+ if (value == null) {
+ value = param.defaultValue;
+ }
+ put(param, value);
+ }
+ }
+
+ /**
+ * @param variableContext
+ * @param parameters
+ */
+ public final static void contextToConfig(IPostParameters variableContext, ConfigParams parameters) {
+ for (ParameterEnum param : CONFIGURATIONLIST) {
+ String p = variableContext.getParameter(param.name().toLowerCase());
+ if (p != null) {
+ parameters.setParameter(param.name(), p);
+ }
+ }
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConfig.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,554 @@
+/* $Id: FileOutputConnector.java 991374 2013-05-31 23:04:08Z minoru $ */
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.manifoldcf.agents.output.hdfs;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URLEncoder;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
+import org.apache.manifoldcf.agents.interfaces.IOutputRemoveActivity;
+import org.apache.manifoldcf.agents.interfaces.OutputSpecification;
+import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
+import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
+import org.apache.manifoldcf.agents.output.BaseOutputConnector;
+import org.apache.manifoldcf.agents.output.hdfs.HDFSOutputParam.ParameterEnum;
+import org.apache.manifoldcf.core.interfaces.ConfigParams;
+import org.apache.manifoldcf.core.interfaces.ConfigurationNode;
+import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
+import org.apache.manifoldcf.core.interfaces.IPostParameters;
+import org.apache.manifoldcf.core.interfaces.IThreadContext;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.core.interfaces.SpecificationNode;
+import org.json.JSONException;
+
+public class HDFSOutputConnector extends BaseOutputConnector {
+
+ public static final String _rcsid = "@(#)$Id: FileOutputConnector.java 988245 2010-08-23 18:39:35Z minoru $";
+
+ // Activities we log
+
+ /** Ingestion activity */
+ public final static String INGEST_ACTIVITY = "document ingest";
+ /** Document removal activity */
+ public final static String REMOVE_ACTIVITY = "document deletion";
+
+ // Activities list
+ protected static final String[] activitiesList = new String[]{INGEST_ACTIVITY, REMOVE_ACTIVITY};
+
+ /** Forward to the javascript to check the configuration parameters */
+ private static final String EDIT_CONFIGURATION_JS = "editConfiguration.js";
+
+ /** Forward to the HTML template to edit the configuration parameters */
+ private static final String EDIT_CONFIGURATION_HTML = "editConfiguration.html";
+
+ /** Forward to the HTML template to view the configuration parameters */
+ private static final String VIEW_CONFIGURATION_HTML = "viewConfiguration.html";
+
+ /** Forward to the javascript to check the specification parameters for the job */
+ private static final String EDIT_SPECIFICATION_JS = "editSpecification.js";
+
+ /** Forward to the template to edit the configuration parameters for the job */
+ private static final String EDIT_SPECIFICATION_HTML = "editSpecification.html";
+
+ /** Forward to the template to view the specification parameters for the job */
+ private static final String VIEW_SPECIFICATION_HTML = "viewSpecification.html";
+
+ protected Configuration config = null;
+ protected FileSystem fileSystem = null;
+
+ /** Constructor.
+ */
+ public HDFSOutputConnector() {
+ }
+
+ /** Return the list of activities that this connector supports (i.e. writes into the log).
+ *@return the list.
+ */
+ @Override
+ public String[] getActivitiesList() {
+ return activitiesList;
+ }
+
+ /** Connect.
+ *@param configParameters is the set of configuration parameters, which
+ * in this case describe the target appliance, basic auth configuration, etc. (This formerly came
+ * out of the ini file.)
+ */
+ @Override
+ public void connect(ConfigParams configParams) {
+ super.connect(configParams);
+
+ String nameNode = configParams.getParameter(ParameterEnum.NAMENODE.name());
+
+ String user = configParams.getParameter(ParameterEnum.USER.name());
+
+ /*
+ * make Configuration
+ */
+ ClassLoader ocl = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(org.apache.hadoop.conf.Configuration.class.getClassLoader());
+ config = new Configuration();
+ config.set("fs.default.name", nameNode);
+ } finally {
+ Thread.currentThread().setContextClassLoader(ocl);
+ }
+
+ /*
+ * get connection to HDFS
+ */
+ try {
+ fileSystem = FileSystem.get(new URI(nameNode), config, user);
+ } catch (URISyntaxException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /** Close the connection. Call this before discarding the connection.
+ */
+ @Override
+ public void disconnect() throws ManifoldCFException {
+ try {
+ fileSystem.close();
+ } catch(IOException ex) {
+ throw new ManifoldCFException(ex);
+ }
+ config.clear();
+ super.disconnect();
+ }
+
+ /** Set up a session */
+ protected void getSession() throws ManifoldCFException, ServiceInterruption {
+ }
+
+ /** Test the connection. Returns a string describing the connection integrity.
+ *@return the connection's status as a displayable string.
+ */
+ @Override
+ public String check() throws ManifoldCFException {
+ try {
+ getSession();
+ return super.check();
+ } catch (ServiceInterruption e) {
+ return "Transient error: "+e.getMessage();
+ }
+ }
+
+ /** Get an output version string, given an output specification. The output version string is used to uniquely describe the pertinent details of
+ * the output specification and the configuration, to allow the Connector Framework to determine whether a document will need to be output again.
+ * Note that the contents of the document cannot be considered by this method, and that a different version string (defined in IRepositoryConnector)
+ * is used to describe the version of the actual document.
+ *
+ * This method presumes that the connector object has been configured, and it is thus able to communicate with the output data store should that be
+ * necessary.
+ *@param spec is the current output specification for the job that is doing the crawling.
+ *@return a string, of unlimited length, which uniquely describes output configuration and specification in such a way that if two such strings are equal,
+ * the document will not need to be sent again to the output data store.
+ */
+ @Override
+ public String getOutputDescription(OutputSpecification spec) throws ManifoldCFException, ServiceInterruption {
+ HDFSOutputSpecs specs = new HDFSOutputSpecs(getSpecNode(spec));
+ return specs.toJson().toString();
+ }
+
+ /** Add (or replace) a document in the output data store using the connector.
+ * This method presumes that the connector object has been configured, and it is thus able to communicate with the output data store should that be
+ * necessary.
+ * The OutputSpecification is *not* provided to this method, because the goal is consistency, and if output is done it must be consistent with the
+ * output description, since that was what was partly used to determine if output should be taking place. So it may be necessary for this method to decode
+ * an output description string in order to determine what should be done.
+ *@param documentURI is the URI of the document. The URI is presumed to be the unique identifier which the output data store will use to process
+ * and serve the document. This URI is constructed by the repository connector which fetches the document, and is thus universal across all output connectors.
+ *@param outputDescription is the description string that was constructed for this document by the getOutputDescription() method.
+ *@param document is the document data to be processed (handed to the output data store).
+ *@param authorityNameString is the name of the authority responsible for authorizing any access tokens passed in with the repository document. May be null.
+ *@param activities is the handle to an object that the implementer of an output connector may use to perform operations, such as logging processing activity.
+ *@return the document status (accepted or permanently rejected).
+ */
+ @Override
+ public int addOrReplaceDocument(String documentURI, String outputDescription, RepositoryDocument document, String authorityNameString, IOutputAddActivity activities) throws ManifoldCFException, ServiceInterruption {
+ // Establish a session
+ getSession();
+
+ HDFSOutputConfig config = getConfigParameters(null);
+
+ HDFSOutputSpecs specs = null;
+ InputStream input = null;
+ FSDataOutputStream output = null;
+ FileLock lock = null;
+ try {
+ specs = new HDFSOutputSpecs(outputDescription);
+
+ /*
+ * make file path
+ */
+ StringBuffer strBuff = new StringBuffer();
+ if (specs.getRootPath() != null) {
+ strBuff.append(specs.getRootPath());
+ }
+ strBuff.append("/");
+ strBuff.append(documentURItoFilePath(documentURI));
+ Path path = new Path(strBuff.toString());
+
+ /*
+ * make directory
+ */
+ if (!fileSystem.exists(path.getParent())) {
+ fileSystem.mkdirs(path.getParent());
+ }
+
+ /*
+ * delete old file
+ */
+ if (fileSystem.exists(path)) {
+ fileSystem.delete(path, true);
+ }
+
+ input = document.getBinaryStream();
+ output = fileSystem.create(path);
+
+ /*
+ * write file
+ */
+ byte buf[] = new byte[1024];
+ int len;
+ while((len = input.read(buf)) != -1) {
+ output.write(buf, 0, len);
+ }
+ output.flush();
+ } catch (JSONException e) {
+ return DOCUMENTSTATUS_REJECTED;
+ } catch (URISyntaxException e) {
+ return DOCUMENTSTATUS_REJECTED;
+ } catch (NullPointerException e) {
+ return DOCUMENTSTATUS_REJECTED;
+ } catch (IOException e) {
+ return DOCUMENTSTATUS_REJECTED;
+ } finally {
+ try {
+ input.close();
+ } catch (IOException e) {
+ }
+ try {
+ output.close();
+ } catch (IOException e) {
+ }
+ }
+
+ activities.recordActivity(null, INGEST_ACTIVITY, new Long(document.getBinaryLength()), documentURI, "OK", null);
+ return DOCUMENTSTATUS_ACCEPTED;
+ }
+
+ /** Remove a document using the connector.
+ * Note that the last outputDescription is included, since it may be necessary for the connector to use such information to know how to properly remove the document.
+ *@param documentURI is the URI of the document. The URI is presumed to be the unique identifier which the output data store will use to process
+ * and serve the document. This URI is constructed by the repository connector which fetches the document, and is thus universal across all output connectors.
+ *@param outputDescription is the last description string that was constructed for this document by the getOutputDescription() method above.
+ *@param activities is the handle to an object that the implementer of an output connector may use to perform operations, such as logging processing activity.
+ */
+ @Override
+ public void removeDocument(String documentURI, String outputDescription, IOutputRemoveActivity activities) throws ManifoldCFException, ServiceInterruption {
+ // Establish a session
+ getSession();
+
+ HDFSOutputConfig config = getConfigParameters(null);
+
+ HDFSOutputSpecs specs = null;
+ try {
+ specs = new HDFSOutputSpecs(outputDescription);
+
+ /*
+ * make path
+ */
+ StringBuffer strBuff = new StringBuffer();
+ if (specs.getRootPath() != null) {
+ strBuff.append(specs.getRootPath());
+ }
+ strBuff.append("/");
+ strBuff.append(documentURItoFilePath(documentURI));
+ Path path = new Path(strBuff.toString());
+
+ /*
+ * delete old file
+ */
+ if (fileSystem.exists(path)) {
+ fileSystem.delete(path, true);
+ }
+ } catch (JSONException e) {
+ } catch (URISyntaxException e) {
+ } catch (NullPointerException e) {
+ } catch (IOException e) {
+ }
+
+ activities.recordActivity(null, REMOVE_ACTIVITY, null, documentURI, "OK", null);
+ }
+
+ /** Output the configuration header section.
+ * This method is called in the head section of the connector's configuration page. Its purpose is to add the required tabs to the list, and to output any
+ * javascript methods that might be needed by the configuration editing HTML.
+ *@param threadContext is the local thread context.
+ *@param out is the output to which any HTML should be sent.
+ *@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
+ *@param tabsArray is an array of tab names. Add to this array any tab names that are specific to the connector.
+ */
+ @Override
+ public void outputConfigurationHeader(IThreadContext threadContext, IHTTPOutput out, Locale locale, ConfigParams parameters, List<String> tabsArray) throws ManifoldCFException, IOException {
+ super.outputConfigurationHeader(threadContext, out, locale, parameters, tabsArray);
+ tabsArray.add(Messages.getString(locale,"HDFSOutputConnector.ServerTabName"));
+ outputResource(EDIT_CONFIGURATION_JS, out, locale, null, null);
+ }
+
+ /** Output the configuration body section.
+ * This method is called in the body section of the connector's configuration page. Its purpose is to present the required form elements for editing.
+ * The coder can presume that the HTML that is output from this configuration will be within appropriate <html>, <body>, and <form> tags. The name of the
+ * form is "editconnection".
+ *@param threadContext is the local thread context.
+ *@param out is the output to which any HTML should be sent.
+ *@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
+ *@param tabName is the current tab name.
+ */
+ @Override
+ public void outputConfigurationBody(IThreadContext threadContext, IHTTPOutput out, Locale locale, ConfigParams parameters, String tabName) throws ManifoldCFException, IOException {
+ super.outputConfigurationBody(threadContext, out, locale, parameters, tabName);
+ HDFSOutputConfig config = this.getConfigParameters(parameters);
+ outputResource(EDIT_CONFIGURATION_HTML, out, locale, config, tabName);
+ }
+
+ /** Process a configuration post.
+ * This method is called at the start of the connector's configuration page, whenever there is a possibility that form data for a connection has been
+ * posted. Its purpose is to gather form information and modify the configuration parameters accordingly.
+ * The name of the posted form is "editconnection".
+ *@param threadContext is the local thread context.
+ *@param variableContext is the set of variables available from the post, including binary file post information.
+ *@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
+ *@return null if all is well, or a string error message if there is an error that should prevent saving of the connection (and cause a redirection to an error page).
+ */
+ @Override
+ public String processConfigurationPost(IThreadContext threadContext, IPostParameters variableContext, Locale locale, ConfigParams parameters) throws ManifoldCFException {
+ HDFSOutputConfig.contextToConfig(variableContext, parameters);
+ return null;
+ }
+
+ /** View configuration.
+ * This method is called in the body section of the connector's view configuration page. Its purpose is to present the connection information to the user.
+ * The coder can presume that the HTML that is output from this configuration will be within appropriate <html> and <body> tags.
+ *@param threadContext is the local thread context.
+ *@param out is the output to which any HTML should be sent.
+ *@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
+ */
+ @Override
+ public void viewConfiguration(IThreadContext threadContext, IHTTPOutput out, Locale locale, ConfigParams parameters) throws ManifoldCFException, IOException {
+ outputResource(VIEW_CONFIGURATION_HTML, out, locale, getConfigParameters(parameters), null);
+ }
+
+ /** Output the specification header section.
+ * This method is called in the head section of a job page which has selected an output connection of the current type. Its purpose is to add the required tabs
+ * to the list, and to output any javascript methods that might be needed by the job editing HTML.
+ *@param out is the output to which any HTML should be sent.
+ *@param os is the current output specification for this job.
+ *@param tabsArray is an array of tab names. Add to this array any tab names that are specific to the connector.
+ */
+ @Override
+ public void outputSpecificationHeader(IHTTPOutput out, Locale locale, OutputSpecification os, List<String> tabsArray) throws ManifoldCFException, IOException {
+ super.outputSpecificationHeader(out, locale, os, tabsArray);
+ tabsArray.add(Messages.getString(locale, "HDFSOutputConnector.PathTabName"));
+ outputResource(EDIT_SPECIFICATION_JS, out, locale, null, null);
+ }
+
+ /** Output the specification body section.
+ * This method is called in the body section of a job page which has selected an output connection of the current type. Its purpose is to present the required form elements for editing.
+ * The coder can presume that the HTML that is output from this configuration will be within appropriate <html>, <body>, and <form> tags. The name of the
+ * form is "editjob".
+ *@param out is the output to which any HTML should be sent.
+ *@param os is the current output specification for this job.
+ *@param tabName is the current tab name.
+ */
+ @Override
+ public void outputSpecificationBody(IHTTPOutput out, Locale locale, OutputSpecification os, String tabName) throws ManifoldCFException, IOException {
+ super.outputSpecificationBody(out, locale, os, tabName);
+ HDFSOutputSpecs specs = getSpecParameters(os);
+ outputResource(EDIT_SPECIFICATION_HTML, out, locale, specs, tabName);
+ }
+
+ /** Process a specification post.
+ * This method is called at the start of job's edit or view page, whenever there is a possibility that form data for a connection has been
+ * posted. Its purpose is to gather form information and modify the output specification accordingly.
+ * The name of the posted form is "editjob".
+ *@param variableContext contains the post data, including binary file-upload information.
+ *@param os is the current output specification for this job.
+ *@return null if all is well, or a string error message if there is an error that should prevent saving of the job (and cause a redirection to an error page).
+ */
+ @Override
+ public String processSpecificationPost(IPostParameters variableContext, Locale locale, OutputSpecification os) throws ManifoldCFException {
+ ConfigurationNode specNode = getSpecNode(os);
+ boolean bAdd = (specNode == null);
+ if (bAdd) {
+ specNode = new SpecificationNode(HDFSOutputConstant.PARAM_ROOTPATH);
+ }
+ HDFSOutputSpecs.contextToSpecNode(variableContext, specNode);
+ if (bAdd) {
+ os.addChild(os.getChildCount(), specNode);
+ }
+
+ return null;
+ }
+
+ /** View specification.
+ * This method is called in the body section of a job's view page. Its purpose is to present the output specification information to the user.
+ * The coder can presume that the HTML that is output from this configuration will be within appropriate <html> and <body> tags.
+ *@param out is the output to which any HTML should be sent.
+ *@param os is the current output specification for this job.
+ */
+ @Override
+ public void viewSpecification(IHTTPOutput out, Locale locale, OutputSpecification os) throws ManifoldCFException, IOException {
+ outputResource(VIEW_SPECIFICATION_HTML, out, locale, getSpecParameters(os), null);
+ }
+
+ /**
+ * @param os
+ * @return
+ */
+ final private SpecificationNode getSpecNode(OutputSpecification os)
+ {
+ int l = os.getChildCount();
+ for (int i = 0; i < l; i++) {
+ SpecificationNode node = os.getChild(i);
+ if (node.getType().equals(HDFSOutputConstant.PARAM_ROOTPATH)) {
+ return node;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * @param os
+ * @return
+ * @throws ManifoldCFException
+ */
+ final private HDFSOutputSpecs getSpecParameters(OutputSpecification os) throws ManifoldCFException {
+ return new HDFSOutputSpecs(getSpecNode(os));
+ }
+
+ /**
+ * @param configParams
+ * @return
+ */
+ final private HDFSOutputConfig getConfigParameters(ConfigParams configParams) {
+ if (configParams == null)
+ configParams = getConfiguration();
+ return new HDFSOutputConfig(configParams);
+ }
+
+ /** Read the content of a resource, replace the variable ${PARAMNAME} with the
+ * value and copy it to the out.
+ *
+ * @param resName
+ * @param out
+ * @throws ManifoldCFException */
+ private static void outputResource(String resName, IHTTPOutput out, Locale locale, HDFSOutputParam params, String tabName) throws ManifoldCFException {
+ Map<String,String> paramMap = null;
+ if (params != null) {
+ paramMap = params.buildMap();
+ if (tabName != null) {
+ paramMap.put("TabName", tabName);
+ }
+ }
+ Messages.outputResourceWithVelocity(out, locale, resName, paramMap, true);
+ }
+
+ /**
+ * @param documentURI
+ * @return
+ * @throws URISyntaxException
+ * @throws NullPointerException
+ */
+ final private String documentURItoFilePath(String documentURI) throws URISyntaxException, NullPointerException {
+ StringBuffer path = new StringBuffer();
+ URI uri = null;
+
+ uri = new URI(documentURI);
+
+ if (uri.getScheme() != null) {
+ path.append(uri.getScheme());
+ path.append("/");
+ }
+
+ if (uri.getHost() != null) {
+ path.append(uri.getHost());
+ if (uri.getPort() != -1) {
+ path.append(":");
+ path.append(uri.getPort());
+ }
+ if (uri.getRawPath() != null) {
+ if (uri.getRawPath().length() == 0) {
+ path.append("/");
+ } else if (uri.getRawPath().equals("/")) {
+ path.append(uri.getRawPath());
+ } else {
+ for (String name : uri.getRawPath().split("/")) {
+ if (name.length() > 0) {
+ path.append("/");
+ path.append(name);
+ }
+ }
+ }
+ }
+ if (uri.getRawQuery() != null) {
+ path.append("?");
+ path.append(uri.getRawQuery());
+ }
+ } else {
+ if (uri.getRawSchemeSpecificPart() != null) {
+ for (String name : uri.getRawSchemeSpecificPart().split("/")) {
+ if (name.length() > 0) {
+ path.append("/");
+ path.append(name);
+ }
+ }
+ }
+ }
+
+ if (path.toString().endsWith("/")) {
+ path.append(".content");
+ }
+ return path.toString();
+ }
+}
Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConstant.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConstant.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConstant.java (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConstant.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,38 @@
+/* $Id: FileOutputConstant.java 991374 2013-05-31 23:01:08Z minoru $ */
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.manifoldcf.agents.output.hdfs;
+
+
+/** Parameters and output data for File output connector.
+ */
+public class HDFSOutputConstant
+{
+ public static final String _rcsid = "@(#)$Id: SolrConfig.java 991374 2010-08-31 22:32:08Z minoru $";
+
+ // Configuration parameters
+
+ /** Name node */
+ public static final String PARAM_NAMENODE = "namenode";
+
+ /** User */
+ public static final String PARAM_USER = "user";
+
+ /** Root path */
+ public static final String PARAM_ROOTPATH = "rootpath";
+}
Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConstant.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputParam.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputParam.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputParam.java (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputParam.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,60 @@
+/* $Id: FileOutputParam.java 1299512 2013-05-31 22:59:38Z minoru $ */
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.manifoldcf.agents.output.hdfs;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.manifoldcf.agents.output.hdfs.HDFSOutputParam.ParameterEnum;
+
+/**
+ * Parameters data for the elasticsearch output connector.
+ */
+public class HDFSOutputParam extends HashMap<ParameterEnum, String>
+{
+
+ /** Parameters constants */
+ public enum ParameterEnum {
+ NAMENODE("hdfs://localhost:9000"),
+ USER(""),
+ ROOTPATH("/tmp");
+
+ final protected String defaultValue;
+
+ private ParameterEnum(String defaultValue) {
+ this.defaultValue = defaultValue;
+ }
+ }
+
+ private static final long serialVersionUID = -140994685772720029L;
+
+ protected HDFSOutputParam(ParameterEnum[] params) {
+ super(params.length);
+ }
+
+ final public Map<String, String> buildMap() {
+ Map<String, String> rval = new HashMap<String, String>();
+ for (Map.Entry<ParameterEnum, String> entry : this.entrySet()) {
+ rval.put(entry.getKey().name(), entry.getValue());
+ }
+ return rval;
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputParam.java
------------------------------------------------------------------------------
svn:mime-type = text/plain