You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/04/07 19:22:04 UTC
svn commit: r1585524 - in /manifoldcf/branches/CONNECTORS-916: ./
connectors/ connectors/amazoncloudsearch/
connectors/amazoncloudsearch/connector/
connectors/amazoncloudsearch/connector/src/
connectors/amazoncloudsearch/connector/src/main/ connectors/...
Author: kwright
Date: Mon Apr 7 17:22:03 2014
New Revision: 1585524
URL: http://svn.apache.org/r1585524
Log:
Commit 2.patch
Added:
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/ (with props)
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/build.xml
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/main/
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/main/java/
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/main/java/org/
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/main/java/org/apache/
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/main/java/org/apache/manifoldcf/
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/main/java/org/apache/manifoldcf/agents/
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/AmazonCloudSearchConnector.java
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/SDFModel.java
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/test/
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/test/java/
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/test/java/org/
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/test/java/org/apache/
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/test/java/org/apache/manifoldcf/
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/test/java/org/apache/manifoldcf/agents/
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/test/java/org/apache/manifoldcf/agents/output/
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/test/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/test/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/tests/
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/test/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/tests/AmazonCloudSearchConnectorTest.java
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/test/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/tests/SDFModelTest.java
manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/pom.xml
Modified:
manifoldcf/branches/CONNECTORS-916/build.xml
manifoldcf/branches/CONNECTORS-916/connectors/pom.xml
Modified: manifoldcf/branches/CONNECTORS-916/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-916/build.xml?rev=1585524&r1=1585523&r2=1585524&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-916/build.xml (original)
+++ manifoldcf/branches/CONNECTORS-916/build.xml Mon Apr 7 17:22:03 2014
@@ -721,7 +721,18 @@
<ant dir="connectors/opensearchserver" target="run-tests-HSQLDB"/>
</target>
-
+ <target name="setup-amazoncloudsearch-connector" depends="build-framework" if="downloaded"/>
+<!--
+ <target name="setup-amazoncloudsearch-connector-tests" depends="build-tests-framework" if="downloaded"/>
+-->
+ <target name="build-amazoncloudsearch-connector" depends="setup-amazoncloudsearch-connector" if="downloaded">
+ <ant dir="connectors/amazoncloudsearch" target="build"/>
+ </target>
+<!--
+ <target name="doc-amazoncloudsearch-connector" depends="setup-amazoncloudsearch-connector" if="downloaded">
+ <ant dir="connectors/amazoncloudsearch" target="doc"/>
+ </target>
+-->
<target name="setup-elasticsearch-connector" depends="build-framework" if="downloaded"/>
@@ -2518,7 +2529,45 @@ Use Apache Forrest version forrest-0.9-d
</antcall>
</target>
- <target name="make-core-deps" depends="download-jira-client,download-google-api-client,download-dropbox-client,download-solrj,download-zookeeper,download-httpcomponents,download-json,download-hsqldb,download-xerces,download-commons,download-elasticsearch-plugin,download-solr-plugins,download-sharepoint-plugins,download-jstl,download-xmlgraphics-commons,download-wstx-asl,download-xmlsec,download-xml-apis,download-wss4j,download-velocity,download-streambuffer,download-stax,download-servlet-api,download-xml-resolver,download-osgi,download-opensaml,download-mimepull,download-mail,download-log4j,download-junit,download-jaxws,download-glassfish,download-jaxb,download-tomcat,download-h2,download-h2-support,download-geronimo-specs,download-fop,download-derby,download-postgresql,download-axis,download-saaj,download-wsdl4j,download-castor,download-jetty,download-slf4j,download-xalan,download-activation,download-avalon-framework,download-poi,download-chemistry,download-ecj,download-hadoop,
download-protobuf">
+ <target name="download-tika">
+ <mkdir dir="lib"/>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/apache/tika"/>
+ <param name="artifact-version" value="1.5"/>
+ <param name="artifact-name" value="tika-core"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/apache/tika"/>
+ <param name="artifact-version" value="1.5"/>
+ <param name="artifact-name" value="tika-parsers"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="org/ccil/cowan/tagsoup"/>
+ <param name="artifact-version" value="1.2.1"/>
+ <param name="artifact-name" value="tagsoup"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ </target>
+
+ <target name="download-jackson">
+ <mkdir dir="lib"/>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="com/fasterxml/jackson/core"/>
+ <param name="artifact-version" value="2.1.3"/>
+ <param name="artifact-name" value="jackson-databind"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven"><param name="target" value="lib"/>
+ <param name="project-path" value="com/fasterxml/jackson/core"/>
+ <param name="artifact-version" value="2.1.2"/>
+ <param name="artifact-name" value="jackson-annotations"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ </target>
+
+ <target name="make-core-deps" depends="download-jira-client,download-google-api-client,download-dropbox-client,download-solrj,download-zookeeper,download-httpcomponents,download-json,download-hsqldb,download-xerces,download-commons,download-elasticsearch-plugin,download-solr-plugins,download-sharepoint-plugins,download-jstl,download-xmlgraphics-commons,download-wstx-asl,download-xmlsec,download-xml-apis,download-wss4j,download-velocity,download-streambuffer,download-stax,download-servlet-api,download-xml-resolver,download-osgi,download-opensaml,download-mimepull,download-mail,download-log4j,download-junit,download-jaxws,download-glassfish,download-jaxb,download-tomcat,download-h2,download-h2-support,download-geronimo-specs,download-fop,download-derby,download-postgresql,download-axis,download-saaj,download-wsdl4j,download-castor,download-jetty,download-slf4j,download-xalan,download-activation,download-avalon-framework,download-poi,download-chemistry,download-ecj,download-hadoop,
download-protobuf,download-tika,download-jackson">
<copy todir="lib">
<fileset dir="lib-license" includes="*.txt"/>
</copy>
Propchange: manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Mon Apr 7 17:22:03 2014
@@ -0,0 +1,10 @@
+build
+dist
+doc
+target
+test-output
+test-derby-output
+test-postgresql-output
+test-HSQLDB-output
+test-HSQLDBext-output
+test-mysql-output
Added: manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/build.xml?rev=1585524&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/build.xml (added)
+++ manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/build.xml Mon Apr 7 17:22:03 2014
@@ -0,0 +1,68 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project name="amazoncloudsearch" default="all">
+
+ <property environment="env"/>
+ <condition property="mcf-dist" value="${env.MCFDISTPATH}">
+ <isset property="env.MCFDISTPATH"/>
+ </condition>
+ <property name="abs-dist" location="../../dist"/>
+ <condition property="mcf-dist" value="${abs-dist}">
+ <not>
+ <isset property="env.MCFDISTPATH"/>
+ </not>
+ </condition>
+
+ <import file="${mcf-dist}/connector-build.xml"/>
+
+ <path id="connector-classpath">
+ <path refid="mcf-connector-build.connector-classpath"/>
+ <fileset dir="../../lib">
+ <include name="httpclient*.jar"/>
+ <include name="jackson-core*.jar"/>
+ <include name="jackson-databind*.jar"/>
+ <include name="jackson-annotations*.jar"/>
+ <include name="tika-core*.jar"/>
+ <include name="tika-parsers*.jar"/>
+ <include name="tagsoup*.jar"/>
+ </fileset>
+ </path>
+
+ <target name="lib" depends="mcf-connector-build.lib,precompile-check" if="canBuild">
+ <mkdir dir="dist/lib"/>
+ <copy todir="dist/lib">
+ <fileset dir="../../lib">
+ <include name="httpclient*.jar"/>
+ <include name="jackson-core*.jar"/>
+ <include name="jackson-databind*.jar"/>
+ <include name="jackson-annotations*.jar"/>
+ <include name="tika-core*.jar"/>
+ <include name="tika-parsers*.jar"/>
+ <include name="tagsoup*.jar"/>
+ </fileset>
+ </copy>
+ </target>
+
+ <target name="deliver-connector" depends="mcf-connector-build.deliver-connector">
+ <antcall target="general-add-output-connector">
+ <param name="connector-label" value="AmazonCloudSearch"/>
+ <param name="connector-class" value="org.apache.manifoldcf.agents.output.amazoncloudsearch.AmazonCloudSearchConnector"/>
+ </antcall>
+ </target>
+
+</project>
Added: manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/AmazonCloudSearchConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/AmazonCloudSearchConnector.java?rev=1585524&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/AmazonCloudSearchConnector.java (added)
+++ manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/AmazonCloudSearchConnector.java Mon Apr 7 17:22:03 2014
@@ -0,0 +1,366 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.agents.output.amazoncloudsearch;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.http.Consts;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.ClientProtocolException;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.DefaultHttpClientConnection;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
+import org.apache.manifoldcf.agents.interfaces.IOutputRemoveActivity;
+import org.apache.manifoldcf.agents.interfaces.OutputSpecification;
+import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
+import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
+import org.apache.manifoldcf.agents.output.BaseOutputConnector;
+import org.apache.manifoldcf.agents.output.amazoncloudsearch.SDFModel.Document;
+import org.apache.manifoldcf.core.interfaces.ConfigParams;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.ParsingReader;
+import org.apache.tika.parser.html.HtmlParser;
+import org.apache.tika.sax.BodyContentHandler;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.JsonToken;
+
+public class AmazonCloudSearchConnector extends BaseOutputConnector {
+
+ /** Ingestion activity */
+ public final static String INGEST_ACTIVITY = "document ingest";
+ /** Document removal activity */
+ public final static String REMOVE_ACTIVITY = "document deletion";
+
+ /** Local connection */
+ protected HttpPost poster = null;
+
+ /** Constructor.
+ */
+ public AmazonCloudSearchConnector(){
+ }
+
+ /** Return the list of activities that this connector supports (i.e. writes into the log).
+ *@return the list.
+ */
+ @Override
+ public String[] getActivitiesList()
+ {
+ return new String[]{INGEST_ACTIVITY,REMOVE_ACTIVITY};
+ }
+
+ /** Connect.
+ *@param configParameters is the set of configuration parameters, which
+ * in this case describe the target appliance, basic auth configuration, etc. (This formerly came
+ * out of the ini file.)
+ */
+ @Override
+ public void connect(ConfigParams configParameters)
+ {
+ super.connect(configParameters);
+ }
+
+ /** This method is called to assess whether to count this connector instance should
+ * actually be counted as being connected.
+ *@return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return poster != null;
+ }
+
+ /** Close the connection. Call this before discarding the connection.
+ */
+ @Override
+ public void disconnect()
+ throws ManifoldCFException
+ {
+ if (poster != null)
+ {
+ poster = null;
+ }
+ super.disconnect();
+ }
+
+ /** Set up a session */
+ protected void getSession()
+ throws ManifoldCFException
+ {
+ //curl -X POST --upload-file data1.json doc.movies-123456789012.us-east-1.cloudsearch.amazonaws.com/2013-01-01/documents/batch --header "Content-Type:application/json"
+ String documentEndpointUrl = "doc-test1-hjzolhfixtfctmuaezbzinjduu.us-east-1.cloudsearch.amazonaws.com";
+ String urlStr = "https://" + documentEndpointUrl + "/2013-01-01/documents/batch";
+ poster = new HttpPost(urlStr);
+
+ //set proxy
+ String proxyHost = System.getenv().get("HTTP_PROXY");
+ if(proxyHost != null)
+ {
+ String host = proxyHost.substring(proxyHost.indexOf("://")+3,proxyHost.lastIndexOf(":"));
+ String port = proxyHost.substring(proxyHost.lastIndexOf(":")+1,proxyHost.length()-1);
+ HttpHost proxy = new HttpHost(host, Integer.parseInt(port), "http");
+ RequestConfig config = RequestConfig.custom().setProxy(proxy).build();
+ poster.setConfig(config);
+ }
+
+ poster.addHeader("Content-Type", "application/json");
+ }
+
+ /** Test the connection. Returns a string describing the connection integrity.
+ *@return the connection's status as a displayable string.
+ */
+ @Override
+ public String check()
+ throws ManifoldCFException
+ {
+ try
+ {
+ getSession();
+ String responsbody = postData("[]");
+
+ String status = "";
+ String message = "";
+ JsonFactory factory = new JsonFactory();
+ JsonParser parser = factory.createJsonParser(responsbody);
+ while (parser.nextToken() != JsonToken.END_OBJECT) {
+ String name = parser.getCurrentName();
+ if("status".equalsIgnoreCase(name)){
+ status = parser.getValueAsString();
+ }else if("errors".equalsIgnoreCase(name)){
+ message = parseMessage(parser);
+ }
+ }
+ if("error".equalsIgnoreCase(status) &&
+ "Encountered unexpected end of file".equals(message)){
+ return "Connection working.";
+ }
+ return "Connection NOT working.";
+ }
+ catch (ClientProtocolException e) {
+ throw new ManifoldCFException(e);
+ } catch (IOException e) {
+ throw new ManifoldCFException(e);
+ }
+ }
+
+ private String parseMessage(JsonParser parser) throws JsonParseException, IOException {
+ while(parser.nextToken() != JsonToken.END_ARRAY){
+ String name = parser.getCurrentName();
+ if("message".equalsIgnoreCase(name)){
+ return parser.getValueAsString();
+ }
+ }
+ return null;
+ }
+
+ /** Get an output version string, given an output specification. The output version string is used to uniquely describe the pertinent details of
+ * the output specification and the configuration, to allow the Connector Framework to determine whether a document will need to be output again.
+ * Note that the contents of the document cannot be considered by this method, and that a different version string (defined in IRepositoryConnector)
+ * is used to describe the version of the actual document.
+ *
+ * This method presumes that the connector object has been configured, and it is thus able to communicate with the output data store should that be
+ * necessary.
+ *@param spec is the current output specification for the job that is doing the crawling.
+ *@return a string, of unlimited length, which uniquely describes output configuration and specification in such a way that if two such strings are equal,
+ * the document will not need to be sent again to the output data store.
+ */
+ @Override
+ public String getOutputDescription(OutputSpecification spec)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ return "";
+ }
+
+
+ /** Detect if a mime type is indexable or not. This method is used by participating repository connectors to pre-filter the number of
+ * unusable documents that will be passed to this output connector.
+ *@param outputDescription is the document's output version.
+ *@param mimeType is the mime type of the document.
+ *@return true if the mime type is indexable by this connector.
+ */
+ public boolean checkMimeTypeIndexable(String outputDescription, String mimeType)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ getSession();
+
+ if(("text/html").equalsIgnoreCase(mimeType)){
+ return super.checkMimeTypeIndexable(outputDescription,mimeType);
+ }
+ return false;
+ }
+
+ /** Add (or replace) a document in the output data store using the connector.
+ * This method presumes that the connector object has been configured, and it is thus able to communicate with the output data store should that be
+ * necessary.
+ * The OutputSpecification is *not* provided to this method, because the goal is consistency, and if output is done it must be consistent with the
+ * output description, since that was what was partly used to determine if output should be taking place. So it may be necessary for this method to decode
+ * an output description string in order to determine what should be done.
+ *@param documentURI is the URI of the document. The URI is presumed to be the unique identifier which the output data store will use to process
+ * and serve the document. This URI is constructed by the repository connector which fetches the document, and is thus universal across all output connectors.
+ *@param outputDescription is the description string that was constructed for this document by the getOutputDescription() method.
+ *@param document is the document data to be processed (handed to the output data store).
+ *@param authorityNameString is the name of the authority responsible for authorizing any access tokens passed in with the repository document. May be null.
+ *@param activities is the handle to an object that the implementer of an output connector may use to perform operations, such as logging processing activity.
+ *@return the document status (accepted or permanently rejected).
+ */
+ @Override
+ public int addOrReplaceDocument(String documentURI, String outputDescription, RepositoryDocument document, String authorityNameString, IOutputAddActivity activities)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ // Establish a session
+ getSession();
+ try {
+ InputStream is = document.getBinaryStream();
+ Parser parser = new HtmlParser();
+ ContentHandler handler = new BodyContentHandler();
+ Metadata metadata = new Metadata();
+ parser.parse(is, handler, metadata, new ParseContext());
+
+ //build json..
+ SDFModel model = new SDFModel();
+ Document doc = model.new Document();
+ doc.setType("add");
+ doc.setId(documentURI);
+
+ //set body text.
+ Map<String,Object> fields = new HashMap<String,Object>();
+ String bodyStr = handler.toString();
+ if(bodyStr != null){
+ bodyStr = handler.toString().replaceAll("\\n", "").replaceAll("\\t", "");
+ fields.put("body", bodyStr);
+ }
+
+ //mapping metadata to SDF fields.
+ String contenttype = metadata.get("Content-Style-Type");
+ String title = metadata.get("dc:title");
+ String size = metadata.get("Content-Length");
+ String description = metadata.get("description");
+ String keywords = metadata.get("keywords");
+ if(contenttype != null && !"".equals(contenttype)) fields.put("content_type", contenttype);
+ if(title != null && !"".equals(title)) fields.put("title", title);
+ if(size != null && !"".equals(size)) fields.put("size", size);
+ if(description != null && !"".equals(description)) fields.put("description", description);
+ if(keywords != null && !"".equals(keywords))
+ {
+ List<String> keywordList = new ArrayList<String>();
+ for(String tmp : keywords.split(",")){
+ keywordList.add(tmp);
+ }
+ fields.put("keywords", keywordList);
+ }
+ doc.setFields(fields);
+ model.addDocument(doc);
+
+ //generate json data.
+ String jsondata = model.toJSON();
+
+ //post data..
+ String responsbody = postData(jsondata);
+
+ activities.recordActivity(null,INGEST_ACTIVITY,new Long(document.getBinaryLength()),documentURI,"OK",null);
+ return DOCUMENTSTATUS_ACCEPTED;
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (SAXException e) {
+ e.printStackTrace();
+ } catch (TikaException e) {
+ e.printStackTrace();
+ }
+ return DOCUMENTSTATUS_REJECTED;
+ }
+
+ /** Remove a document using the connector.
+ * Note that the last outputDescription is included, since it may be necessary for the connector to use such information to know how to properly remove the document.
+ *@param documentURI is the URI of the document. The URI is presumed to be the unique identifier which the output data store will use to process
+ * and serve the document. This URI is constructed by the repository connector which fetches the document, and is thus universal across all output connectors.
+ *@param outputDescription is the last description string that was constructed for this document by the getOutputDescription() method above.
+ *@param activities is the handle to an object that the implementer of an output connector may use to perform operations, such as logging processing activity.
+ */
+ @Override
+ public void removeDocument(String documentURI, String outputDescription, IOutputRemoveActivity activities)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ // Establish a session
+ getSession();
+
+ String jsonData = "";
+ try {
+ SDFModel model = new SDFModel();
+ SDFModel.Document doc = model.new Document();
+ doc.setType("delete");
+ doc.setId(documentURI);
+ model.addDocument(doc);
+ jsonData = model.toJSON();
+ } catch (JsonProcessingException e) {
+ throw new ManifoldCFException(e);
+ }
+ String responsbody = postData(jsonData);
+
+
+ activities.recordActivity(null,REMOVE_ACTIVITY,null,documentURI,"OK",null);
+ }
+
+ private String postData(String jsonData) throws ManifoldCFException {
+ CloseableHttpClient httpclient = HttpClients.createDefault();
+ try {
+ poster.setEntity(new StringEntity(jsonData, Consts.UTF_8));
+ HttpResponse res = httpclient.execute(poster);
+
+ HttpEntity resEntity = res.getEntity();
+ return EntityUtils.toString(resEntity);
+
+ } catch (ClientProtocolException e) {
+ throw new ManifoldCFException(e);
+ } catch (IOException e) {
+ throw new ManifoldCFException(e);
+ } finally {
+ try {
+ httpclient.close();
+ } catch (IOException e) {
+ //do nothing
+ }
+ }
+ }
+
+}
\ No newline at end of file
Added: manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/SDFModel.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/SDFModel.java?rev=1585524&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/SDFModel.java (added)
+++ manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/SDFModel.java Mon Apr 7 17:22:03 2014
@@ -0,0 +1,75 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.agents.output.amazoncloudsearch;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize.Inclusion;
+
+public class SDFModel {
+
+ private List<Document> documentList = new ArrayList<Document>();
+
+ public void addDocument(Document doc){
+ documentList.add(doc);
+ }
+
+ public String toJSON() throws JsonProcessingException{
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.setSerializationInclusion(Include.NON_NULL);
+ return mapper.writeValueAsString(documentList);
+ }
+
+ public class Document {
+ private String type;
+ private String id;
+ private Map<String,Object> fields;
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public Map getFields() {
+ return fields;
+ }
+
+ public void setFields(Map<String,Object> fields) {
+ this.fields = fields;
+ }
+ }
+}
Added: manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/test/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/tests/AmazonCloudSearchConnectorTest.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/test/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/tests/AmazonCloudSearchConnectorTest.java?rev=1585524&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/test/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/tests/AmazonCloudSearchConnectorTest.java (added)
+++ manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/test/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/tests/AmazonCloudSearchConnectorTest.java Mon Apr 7 17:22:03 2014
@@ -0,0 +1,111 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.agents.output.amazoncloudsearch.tests;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.manifoldcf.agents.output.amazoncloudsearch.SDFModel;
+import org.apache.manifoldcf.agents.output.amazoncloudsearch.SDFModel.Document;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.html.HtmlParser;
+import org.apache.tika.sax.BodyContentHandler;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+
+public class AmazonCloudSearchConnectorTest {
+
+ public static void main(String[] args){
+ InputStream is;
+ try {
+
+
+
+ is = new FileInputStream(new File("000407.htm"));
+ Parser parser = new HtmlParser();
+ ContentHandler handler = new BodyContentHandler();
+ Metadata metadata = new Metadata();
+ parser.parse(is, handler, metadata, new ParseContext());
+
+ //build json..
+ SDFModel model = new SDFModel();
+ Document doc = model.new Document();
+ doc.setType("add");
+ doc.setId("aabbcc");
+
+ //set body text.
+ Map<String,Object> fields = new HashMap<String,Object>();
+ String bodyStr = handler.toString();
+ if(bodyStr != null){
+ bodyStr = handler.toString().replaceAll("\\n", "").replaceAll("\\t", "");
+ fields.put("body", bodyStr);
+ }
+
+ //mapping metadata to SDF fields.
+ String contenttype = metadata.get("Content-Style-Type");
+ String title = metadata.get("dc.title");
+ String size = metadata.get("Content-Length");
+ String description = metadata.get("description");
+ String keywords = metadata.get("keywords");
+ if(contenttype != null && !"".equals(contenttype)) fields.put("content_type", contenttype);
+ if(title != null && !"".equals(title)) fields.put("title", title);
+ if(size != null && !"".equals(size)) fields.put("size", size);
+ if(description != null && !"".equals(description)) fields.put("description", description);
+ if(keywords != null && !"".equals(keywords))
+ {
+ List<String> keywordList = new ArrayList<String>();
+ for(String tmp : keywords.split(",")){
+ keywordList.add(tmp);
+ }
+ fields.put("keywords", keywordList);
+ }
+ doc.setFields(fields);
+ model.addDocument(doc);
+
+ //generate json data.
+ String jsondata = model.toJSON();
+ System.out.println(jsondata);
+
+ } catch (FileNotFoundException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (SAXException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (TikaException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
+ }
+
+}
Added: manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/test/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/tests/SDFModelTest.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/test/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/tests/SDFModelTest.java?rev=1585524&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/test/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/tests/SDFModelTest.java (added)
+++ manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/connector/src/test/java/org/apache/manifoldcf/agents/output/amazoncloudsearch/tests/SDFModelTest.java Mon Apr 7 17:22:03 2014
@@ -0,0 +1,66 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.agents.output.amazoncloudsearch.tests;
+
+import static org.junit.Assert.*;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.manifoldcf.agents.output.amazoncloudsearch.SDFModel;
+import org.junit.Test;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+
+public class SDFModelTest {
+
+ @Test
+ public void testToJSON() {
+ SDFModel model = new SDFModel();
+
+ SDFModel.Document doc = model.new Document();
+ doc.setType("add");
+ doc.setId("aaaabbbbcccc");
+ Map fields = new HashMap();
+ fields.put("title", "The Seeker: The Dark Is Rising");
+ fields.put("director", "Cunningham, David L.");
+ String[] genre = {"Adventure","Drama","Fantasy","Thriller"};
+ fields.put("genre", genre);
+ doc.setFields(fields);
+
+ model.addDocument(doc);
+
+ SDFModel.Document doc2 = model.new Document();
+ doc2.setType("delete");
+ doc2.setId("xxxxxffffddddee");
+ model.addDocument(doc2);
+
+ try {
+ String jsonStr = model.toJSON();
+ System.out.println(jsonStr);
+ String expect = "[{\"type\":\"add\",\"id\":\"aaaabbbbcccc\",\"fields\":{\"genre\":[\"Adventure\",\"Drama\",\"Fantasy\",\"Thriller\"],\"title\":\"The Seeker: The Dark Is Rising\",\"director\":\"Cunningham, David L.\"}},{\"type\":\"delete\",\"id\":\"xxxxxffffddddee\"}]";
+ assertEquals(expect, jsonStr);
+
+ } catch (JsonProcessingException e) {
+ e.printStackTrace();
+ fail();
+ }
+ }
+
+}
Added: manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/pom.xml
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/pom.xml?rev=1585524&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/pom.xml (added)
+++ manifoldcf/branches/CONNECTORS-916/connectors/amazoncloudsearch/pom.xml Mon Apr 7 17:22:03 2014
@@ -0,0 +1,397 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.manifoldcf</groupId>
+ <artifactId>mcf-connectors</artifactId>
+ <version>1.6-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <developers>
+ <developer>
+ <name>Takumi Yoshida</name>
+ <url>http://yoshi0309.hatenablog.com/</url>
+ </developer>
+ </developers>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ </properties>
+
+ <artifactId>mcf-amazoncloudsearch-connector</artifactId>
+ <name>ManifoldCF - Connectors - Amazon CloudSearch Output</name>
+
+ <build>
+ <defaultGoal>integration-test</defaultGoal>
+ <sourceDirectory>${basedir}/connector/src/main/java</sourceDirectory>
+ <testSourceDirectory>${basedir}/connector/src/test/java</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>${basedir}/connector/src/main/native2ascii</directory>
+ <includes>
+ <include>**/*.properties</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${basedir}/connector/src/main/resources</directory>
+ <includes>
+ <include>**/*.html</include>
+ <include>**/*.js</include>
+ </includes>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>${basedir}/connector/src/test/resources</directory>
+ </testResource>
+ </testResources>
+
+ <plugins>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>native2ascii-maven-plugin</artifactId>
+ <version>1.0-beta-1</version>
+ <configuration>
+ <workDir>target/classes</workDir>
+ </configuration>
+ <executions>
+ <execution>
+ <id>native2ascii-utf8</id>
+ <goals>
+ <goal>native2ascii</goal>
+ </goals>
+ <configuration>
+ <encoding>UTF8</encoding>
+ <includes>
+ <include>**/*.properties</include>
+ </includes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- Test plugin configuration -->
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-war</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/dependency</outputDirectory>
+ <artifactItems>
+ <artifactItem>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>mcf-api-service</artifactId>
+ <version>${project.version}</version>
+ <type>war</type>
+ <overWrite>false</overWrite>
+ <destFileName>mcf-api-service.war</destFileName>
+ </artifactItem>
+ <artifactItem>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>mcf-authority-service</artifactId>
+ <version>${project.version}</version>
+ <type>war</type>
+ <overWrite>false</overWrite>
+ <destFileName>mcf-authority-service.war</destFileName>
+ </artifactItem>
+ <artifactItem>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>mcf-crawler-ui</artifactId>
+ <version>${project.version}</version>
+ <type>war</type>
+ <overWrite>false</overWrite>
+ <destFileName>mcf-crawler-ui.war</destFileName>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>**/*Postgresql*.java</exclude>
+ <exclude>**/*MySQL*.java</exclude>
+ </excludes>
+ <forkMode>always</forkMode>
+ <workingDirectory>target/test-output</workingDirectory>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>2.12.3</version>
+ <configuration>
+ <skipTests>${skipITs}</skipTests>
+ <systemPropertyVariables>
+ <crawlerWarPath>../dependency/mcf-crawler-ui.war</crawlerWarPath>
+ <authorityserviceWarPath>../dependency/mcf-authority-service.war</authorityserviceWarPath>
+ <apiWarPath>../dependency/mcf-api-service.war</apiWarPath>
+ </systemPropertyVariables>
+ <excludes>
+ <exclude>**/*Postgresql*.java</exclude>
+ <exclude>**/*MySQL*.java</exclude>
+ </excludes>
+ <forkMode>always</forkMode>
+ <workingDirectory>target/test-output</workingDirectory>
+ </configuration>
+ <executions>
+ <execution>
+ <id>integration-test</id>
+ <goals>
+ <goal>integration-test</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>verify</id>
+ <goals>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>mcf-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>mcf-agents</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>mcf-ui-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpcomponent.httpclient.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>2.1.3</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.3.2</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>2.3.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>1.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-parsers</artifactId>
+ <version>1.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.ccil.cowan.tagsoup</groupId>
+ <artifactId>tagsoup</artifactId>
+ <version>1.2.1</version>
+ </dependency>
+
+
+ <!-- Testing dependencies -->
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>mcf-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>mcf-agents</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>mcf-pull-agent</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ <version>${postgresql.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.hsqldb</groupId>
+ <artifactId>hsqldb</artifactId>
+ <version>${hsqldb.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derby</artifactId>
+ <version>${derby.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>${mysql.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>mcf-api-service</artifactId>
+ <version>${project.version}</version>
+ <type>war</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>mcf-authority-service</artifactId>
+ <version>${project.version}</version>
+ <type>war</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>mcf-crawler-ui</artifactId>
+ <version>${project.version}</version>
+ <type>war</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-webapp</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-http</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-io</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-security</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-continuation</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-xml</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-api-2.1-glassfish</artifactId>
+ <version>${glassfish.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-2.1-glassfish</artifactId>
+ <version>${glassfish.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+</project>
Modified: manifoldcf/branches/CONNECTORS-916/connectors/pom.xml
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-916/connectors/pom.xml?rev=1585524&r1=1585523&r2=1585524&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-916/connectors/pom.xml (original)
+++ manifoldcf/branches/CONNECTORS-916/connectors/pom.xml Mon Apr 7 17:22:03 2014
@@ -57,6 +57,7 @@
<module>generic</module>
<module>regexpmapper</module>
<module>email</module>
+ <module>amazoncloudsearch</module>
</modules>
</project>