You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by sh...@apache.org on 2015/07/04 09:34:48 UTC
svn commit: r1689113 [1/2] - in /manifoldcf/branches/CONNECTORS-1219: ./
connectors/ connectors/lucene/ connectors/lucene/connector/
connectors/lucene/connector/src/ connectors/lucene/connector/src/main/
connectors/lucene/connector/src/main/java/ conne...
Author: shinichiro
Date: Sat Jul 4 07:34:47 2015
New Revision: 1689113
URL: http://svn.apache.org/r1689113
Log:
initial commit. CONNECTORS-1219-v0.2.patch
Added:
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/build.xml (with props)
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClient.java (with props)
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClientManager.java (with props)
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConfig.java (with props)
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConnector.java (with props)
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneDocument.java (with props)
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/Messages.java (with props)
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_en_US.properties (with props)
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_ja_JP.properties (with props)
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_zh_CN.properties (with props)
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/lucene/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/lucene/editConfiguration.js
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/lucene/editConfiguration_Parameters.html (with props)
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/lucene/viewConfiguration.html (with props)
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/test/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/test/java/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/test/java/org/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/test/java/org/apache/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/test/java/org/apache/manifoldcf/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/test/java/org/apache/manifoldcf/agents/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/test/java/org/apache/manifoldcf/agents/output/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/test/java/org/apache/manifoldcf/agents/output/lucene/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/test/java/org/apache/manifoldcf/agents/output/lucene/tests/
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/test/java/org/apache/manifoldcf/agents/output/lucene/tests/BaseUIHSQLDB.java (with props)
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/test/java/org/apache/manifoldcf/agents/output/lucene/tests/LuceneClientTest.java (with props)
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/test/java/org/apache/manifoldcf/agents/output/lucene/tests/NavigationHSQLDBUI.java (with props)
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/pom.xml (with props)
Modified:
manifoldcf/branches/CONNECTORS-1219/build.xml
manifoldcf/branches/CONNECTORS-1219/connectors/pom.xml
manifoldcf/branches/CONNECTORS-1219/pom.xml
Modified: manifoldcf/branches/CONNECTORS-1219/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/build.xml?rev=1689113&r1=1689112&r2=1689113&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/build.xml (original)
+++ manifoldcf/branches/CONNECTORS-1219/build.xml Sat Jul 4 07:34:47 2015
@@ -128,6 +128,7 @@
<property name="jackson-annotations.version" value="2.1.2"/>
<property name="resteasy.client.version" value="3.0.8.Final"/>
<property name="jsoup.version" value="1.8.1"/>
+ <property name="lucene.version" value="5.2.1"/>
<target name="downloaded-condition">
@@ -1694,8 +1695,7 @@ Use Apache Forrest version forrest-0.9-d
<param name="artifact-type" value="jar"/>
</antcall>
</target>
-
-
+
<target name="download-tika">
<mkdir dir="lib"/>
<antcall target="download-via-maven"><param name="target" value="lib"/>
@@ -1950,8 +1950,47 @@ Use Apache Forrest version forrest-0.9-d
<param name="artifact-type" value="jar"/>
</antcall>
</target>
+
+ <target name="download-lucene">
+ <mkdir dir="lib"/>
+ <antcall target="download-via-maven">
+ <param name="target" value="lib"/>
+ <param name="project-path" value="org/apache/lucene"/>
+ <param name="artifact-version" value="${lucene.version}"/>
+ <param name="artifact-name" value="lucene-core"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven">
+ <param name="target" value="lib"/>
+ <param name="project-path" value="org/apache/lucene"/>
+ <param name="artifact-version" value="${lucene.version}"/>
+ <param name="artifact-name" value="lucene-analyzers-common"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven">
+ <param name="target" value="lib"/>
+ <param name="project-path" value="org/apache/lucene"/>
+ <param name="artifact-version" value="${lucene.version}"/>
+ <param name="artifact-name" value="lucene-queryparser"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven">
+ <param name="target" value="lib"/>
+ <param name="project-path" value="org/apache/lucene"/>
+ <param name="artifact-version" value="${lucene.version}"/>
+ <param name="artifact-name" value="lucene-queries"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ <antcall target="download-via-maven">
+ <param name="target" value="lib"/>
+ <param name="project-path" value="org/apache/lucene"/>
+ <param name="artifact-version" value="${lucene.version}"/>
+ <param name="artifact-name" value="lucene-sandbox"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
+ </target>
- <target name="make-core-deps" depends="download-resteasy,download-jsoup,download-mockito,download-alfresco-webscript-plugin,download-alfresco-indexer-client,download-mongo-java-driver,download-jira-client,download-google-api-client,download-dropbox-client,download-solrj,download-zookeeper,download-httpcomponents,download-json,download-hsqldb,download-xerces,download-commons,download-elasticsearch-plugin,download-solr-plugins,download-sharepoint-plugins,download-jstl,download-xmlgraphics-commons,download-woodstox,download-xmlsec,download-xml-apis,download-wss4j,download-velocity,download-streambuffer,download-stax,download-servlet-api,download-xml-resolver,download-osgi,download-opensaml,download-mimepull,download-mail,download-log4j,download-junit,download-jaxws,download-glassfish,download-jaxb,download-tomcat,download-h2,download-h2-support,download-geronimo-specs,download-fop,download-postgresql,download-axis,download-saaj,download-wsdl4j,download-castor,download-jetty,downloa
d-slf4j,download-xalan,download-activation,download-avalon-framework,download-poi,download-chemistry,download-ecj,download-hadoop,download-protobuf,download-tika,download-jackson">
+ <target name="make-core-deps" depends="download-resteasy,download-jsoup,download-mockito,download-alfresco-webscript-plugin,download-alfresco-indexer-client,download-mongo-java-driver,download-jira-client,download-google-api-client,download-dropbox-client,download-solrj,download-zookeeper,download-httpcomponents,download-json,download-hsqldb,download-xerces,download-commons,download-elasticsearch-plugin,download-solr-plugins,download-sharepoint-plugins,download-jstl,download-xmlgraphics-commons,download-woodstox,download-xmlsec,download-xml-apis,download-wss4j,download-velocity,download-streambuffer,download-stax,download-servlet-api,download-xml-resolver,download-osgi,download-opensaml,download-mimepull,download-mail,download-log4j,download-junit,download-jaxws,download-glassfish,download-jaxb,download-tomcat,download-h2,download-h2-support,download-geronimo-specs,download-fop,download-postgresql,download-axis,download-saaj,download-wsdl4j,download-castor,download-jetty,downloa
d-slf4j,download-xalan,download-activation,download-avalon-framework,download-poi,download-chemistry,download-ecj,download-hadoop,download-protobuf,download-tika,download-jackson,download-lucene">
<copy todir="lib">
<fileset dir="lib-license" includes="*.txt"/>
</copy>
Added: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/build.xml?rev=1689113&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/build.xml (added)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/build.xml Sat Jul 4 07:34:47 2015
@@ -0,0 +1,68 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project name="lucene" default="all">
+
+ <property environment="env"/>
+ <condition property="mcf-dist" value="${env.MCFDISTPATH}">
+ <isset property="env.MCFDISTPATH"/>
+ </condition>
+ <property name="abs-dist" location="../../dist"/>
+ <condition property="mcf-dist" value="${abs-dist}">
+ <not>
+ <isset property="env.MCFDISTPATH"/>
+ </not>
+ </condition>
+
+ <import file="${mcf-dist}/connector-build.xml"/>
+
+ <path id="connector-classpath">
+ <path refid="mcf-connector-build.connector-classpath"/>
+ <fileset dir="../../lib">
+ <include name="lucene-core*.jar"/>
+ <include name="lucene-analyzers-common*.jar"/>
+ <include name="lucene-queryparser*.jar"/>
+ <include name="lucene-queries*.jar"/>
+ <include name="lucene-sandbox*.jar"/>
+ <include name="guava-*.jar"/>
+ <include name="gson*.jar"/>
+ </fileset>
+ </path>
+
+ <target name="lib" depends="mcf-connector-build.lib,precompile-check" if="canBuild">
+ <mkdir dir="dist/lib"/>
+ <copy todir="dist/lib">
+ <fileset dir="../../lib">
+ <include name="lucene-core*.jar"/>
+ <include name="lucene-analyzers-common*.jar"/>
+ <include name="lucene-queryparser*.jar"/>
+ <include name="lucene-queries*.jar"/>
+ <include name="lucene-sandbox*.jar"/>
+ <include name="guava-*.jar"/>
+ <include name="gson*.jar"/>
+ </fileset>
+ </copy>
+ </target>
+
+ <target name="deliver-connector" depends="mcf-connector-build.deliver-connector">
+ <antcall target="general-add-output-connector">
+ <param name="connector-label" value="Lucene"/>
+ <param name="connector-class" value="org.apache.manifoldcf.agents.output.lucene.LuceneConnector"/>
+ </antcall>
+ </target>
+
+</project>
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/build.xml
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/build.xml
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClient.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClient.java?rev=1689113&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClient.java (added)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClient.java Sat Jul 4 07:34:47 2015
@@ -0,0 +1,436 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.agents.output.lucene;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.custom.CustomAnalyzer;
+import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.SlowCompositeReaderWrapper;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.queryparser.flexible.core.QueryNodeException;
+import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.NRTCachingDirectory;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.StandardSystemProperty;
+import com.google.common.collect.Maps;
+import com.google.common.util.concurrent.AbstractScheduledService;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+
+public class LuceneClient implements Closeable {
+
+ private final Path path;
+ private final Map<String,Map<String,Object>> charfiltersInfo;
+ private final Map<String,Map<String,Object>> tokenizersInfo;
+ private final Map<String,Map<String,Object>> filtersInfo;
+ private final Map<String,Map<String,Object>> analyzersInfo;
+ private final Map<String,Map<String,Object>> fieldsInfo;
+ private final String idField;
+ private final String contentField;
+
+ private final String versionString;
+
+ private final IndexWriter writer;
+ private final CommitService commitService;
+ private static final long COMMIT_DELAY_MS = 15000L;
+
+ private DirectoryReader realtimeReader;
+ private final CommitService refreshService;
+ private static final long REFRESH_DELAY_MS = 1000L;
+
+ private final StandardQueryParser queryParser;
+
+ private static final Gson GSON = new Gson();
+ private static final Type TYPE = new TypeToken<Map<String,Map<String,Object>>>(){}.getType();
+
+ /** analyzer attribute */
+ private static final String ATTR_CHARFILTER = "charfilter";
+ private static final String ATTR_TOKENIZER = "tokenizer";
+ private static final String ATTR_FILTER = "filter";
+
+ private static final String ATTR_TYPE = "type";
+ private static final String ATTR_PARAMS = "params";
+
+ /** field attribute */
+ public static final String ATTR_FIELDTYPE = "type";
+ public static final String ATTR_STORE = "store";
+ public static final String ATTR_INDEX_ANALYZER = "index_analyzer";
+ public static final String ATTR_QUERY_ANALYZER = "query_analyzer";
+ public static final String ATTR_COPY_TO = "copy_to";
+
+ public static final String FIELDTYPE_STRING = "string";
+ public static final String FIELDTYPE_TEXT = "text";
+
+ public LuceneClient(Path path) throws IOException {
+ this(path,
+ LuceneClient.defaultCharfilters(), LuceneClient.defaultTokenizers(), LuceneClient.defaultFilters(),
+ LuceneClient.defaultAnalyzers(), LuceneClient.defaultFields(),
+ LuceneClient.defaultIdField(), LuceneClient.defaultContentField());
+ }
+
+ public LuceneClient(Path path,
+ String charfilters, String tokenizers, String filters,
+ String analyzers, String fields,
+ String idField, String contentField) throws IOException {
+ this.path = Preconditions.checkNotNull(path);
+ this.charfiltersInfo = parseAsMap(Preconditions.checkNotNull(charfilters));
+ this.tokenizersInfo = parseAsMap(Preconditions.checkNotNull(tokenizers));
+ this.filtersInfo = parseAsMap(Preconditions.checkNotNull(filters));
+ this.analyzersInfo = parseAsMap(Preconditions.checkNotNull(analyzers));
+ this.fieldsInfo = parseAsMap(Preconditions.checkNotNull(fields));
+ this.idField = Preconditions.checkNotNull(idField);
+ this.contentField = Preconditions.checkNotNull(contentField);
+
+ this.versionString = createVersionString(path, charfiltersInfo, tokenizersInfo, filtersInfo, analyzersInfo, fieldsInfo, idField, contentField);
+
+ Map<String,Analyzer> analyzersMap = createAnalyzersMap();
+ Map<String,Analyzer> fieldIndexAnalyzers = createFieldAnalyzers(analyzersMap, ATTR_INDEX_ANALYZER);
+ Map<String,Analyzer> fieldQueryAnalyzers = createFieldAnalyzers(analyzersMap, ATTR_QUERY_ANALYZER);
+
+ Analyzer indexAnalyzer = new PerFieldAnalyzerWrapper(new KeywordAnalyzer(), fieldIndexAnalyzers);
+ Analyzer queryAnalyzer = new PerFieldAnalyzerWrapper(new KeywordAnalyzer(), fieldQueryAnalyzers);
+
+ IndexWriterConfig config
+ = new IndexWriterConfig(indexAnalyzer)
+ .setOpenMode(OpenMode.CREATE_OR_APPEND)
+ .setUseCompoundFile(false)
+ .setCommitOnClose(IndexWriterConfig.DEFAULT_COMMIT_ON_CLOSE)
+ .setRAMBufferSizeMB(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB * 6);
+
+ Directory fsDir = FSDirectory.open(path);
+ NRTCachingDirectory cachedDir = new NRTCachingDirectory(fsDir, 4, 48);
+
+ this.writer = new IndexWriter(cachedDir, config);
+
+ initIndex();
+
+ this.commitService = new CommitService(this, false, COMMIT_DELAY_MS);
+ commitService.startAsync().awaitRunning();
+
+ this.refreshService = new CommitService(this, true, REFRESH_DELAY_MS);
+ refreshService.startAsync().awaitRunning();
+
+ this.queryParser = new StandardQueryParser(queryAnalyzer);
+ }
+
+ public static Map<String,Map<String,Object>> parseAsMap(String json) {
+ return GSON.fromJson(json, TYPE);
+ }
+
+ private Map<String,Analyzer> createAnalyzersMap() throws IOException {
+ Map<String,Analyzer> analyzersMap = Maps.newHashMap();
+ for (Map.Entry<String,Map<String,Object>> analyzerInfo : analyzersInfo.entrySet()) {
+ String name = analyzerInfo.getKey();
+
+ Map<String,Object> info = analyzerInfo.getValue();
+ @SuppressWarnings("unchecked")
+ final List<String> charfilterList = (List<String>)Objects.firstNonNull(info.get(ATTR_CHARFILTER), new ArrayList<String>());
+ final String tokenizer = info.get(ATTR_TOKENIZER).toString();
+ @SuppressWarnings("unchecked")
+ final List<String> filterList = (List<String>)Objects.firstNonNull(info.get(ATTR_FILTER), new ArrayList<String>());
+
+ CustomAnalyzer.Builder builder = CustomAnalyzer.builder();
+
+ Map<String,Map<String,Object>> charfilterInfo = Maps.filterKeys(charfiltersInfo, new Predicate<String>() {
+ @Override public boolean apply(String key) {
+ return charfilterList.contains(key);
+ }
+ });
+ for (Map.Entry<String,Map<String,Object>> e : charfilterInfo.entrySet()) {
+ String type = e.getValue().get(ATTR_TYPE).toString();
+ @SuppressWarnings("unchecked")
+ Map<String,String> params = (Map<String,String>)Objects.firstNonNull(e.getValue().get(ATTR_PARAMS),new HashMap<String,String>());
+ builder = builder.addCharFilter(type, newMap(params));
+ }
+
+ Map<String,Map<String,Object>> tokenizerInfo = Maps.filterKeys(tokenizersInfo, new Predicate<String>() {
+ @Override public boolean apply(String key) {
+ return tokenizer.equals(key);
+ }
+ });
+ assert tokenizerInfo.size() == 1;
+ for (Map.Entry<String,Map<String,Object>> e : tokenizerInfo.entrySet()) {
+ String type = e.getValue().get(ATTR_TYPE).toString();
+ @SuppressWarnings("unchecked")
+ Map<String,String> params = (Map<String,String>)Objects.firstNonNull(e.getValue().get(ATTR_PARAMS),new HashMap<String,String>());
+ builder = builder.withTokenizer(type, newMap(params));
+ }
+
+ Map<String,Map<String,Object>> filterInfo = Maps.filterKeys(filtersInfo, new Predicate<String>() {
+ @Override public boolean apply(String key) {
+ return filterList.contains(key);
+ }
+ });
+ for (Map.Entry<String,Map<String,Object>> e : filterInfo.entrySet()) {
+ String type = e.getValue().get(ATTR_TYPE).toString();
+ @SuppressWarnings("unchecked")
+ Map<String,String> params = (Map<String,String>)Objects.firstNonNull(e.getValue().get(ATTR_PARAMS),new HashMap<String,String>());
+ builder = builder.addTokenFilter(type, newMap(params));
+ }
+
+ builder = builder.withPositionIncrementGap(100);
+
+ analyzersMap.put(name, builder.build());
+ }
+ return analyzersMap;
+ }
+
+ private Map<String,Analyzer> createFieldAnalyzers(Map<String,Analyzer> analyzersMap, String target) throws IOException {
+ Map<String,Analyzer> fieldAnalyzers = Maps.newHashMap();
+ for (Map.Entry<String,Map<String,Object>> e : fieldsInfo.entrySet()) {
+ if (e.getValue().get(ATTR_FIELDTYPE).toString().equals(FIELDTYPE_TEXT)) {
+ String field = e.getKey();
+ String analyzer = e.getValue().get(target).toString();
+ fieldAnalyzers.put(field, analyzersMap.get(analyzer));
+ }
+ }
+ return fieldAnalyzers;
+ }
+
+ private Map<String,String> newMap(Map<String,String> map) {
+ Map<String,String> copy = Maps.newHashMap();
+ for (Map.Entry<String,String> e : map.entrySet()) {
+ copy.put(e.getKey(), e.getValue());
+ }
+ return copy;
+ }
+
+ private void initIndex() throws IOException {
+ File dirFile = path.toFile();
+ boolean indexExists = dirFile.canRead() && dirFile.list().length > 1;
+ if (!indexExists) writer.commit();
+ }
+
+ public Map<String,Map<String,Object>> fieldsInfo() {
+ return fieldsInfo;
+ }
+
+ public String idField() {
+ return idField;
+ }
+
+ public String contentField() {
+ return contentField;
+ }
+
+ public String versionString() {
+ return versionString;
+ }
+
+ public static String createVersionString(
+ Path path,
+ Map<String,Map<String,Object>> charfiltersInfo,
+ Map<String,Map<String,Object>> tokenizersInfo,
+ Map<String,Map<String,Object>> filtersInfo,
+ Map<String,Map<String,Object>> analyzersInfo,
+ Map<String,Map<String,Object>> fieldsInfo,
+ String idField,String contentField) {
+ return LuceneConfig.PARAM_PATH + ":" + path.toString() + "+"
+ + LuceneConfig.PARAM_CHARFILTERS + ":" + Joiner.on(",").withKeyValueSeparator("=").join(charfiltersInfo) + "+"
+ + LuceneConfig.PARAM_TOKENIZERS + ":" + Joiner.on(",").withKeyValueSeparator("=").join(tokenizersInfo) + "+"
+ + LuceneConfig.PARAM_FILTERS + ":" + Joiner.on(",").withKeyValueSeparator("=").join(filtersInfo) + "+"
+ + LuceneConfig.PARAM_ANALYZERS + ":" + Joiner.on(",").withKeyValueSeparator("=").join(analyzersInfo) + "+"
+ + LuceneConfig.PARAM_FIELDS + ":" + Joiner.on(",").withKeyValueSeparator("=").join(fieldsInfo) + "+"
+ + LuceneConfig.PARAM_IDFIELD + ":" + idField + "+"
+ + LuceneConfig.PARAM_CONTENTFIELD + ":" + contentField;
+ }
+
+ public void refresh() throws IOException {
+ if (realtimeReader == null) {
+ realtimeReader = DirectoryReader.open(writer.getDirectory());
+ }
+ DirectoryReader newReader = DirectoryReader.openIfChanged(realtimeReader, writer, true);
+ if (newReader != null) realtimeReader = newReader;
+ }
+
+ public void commit() throws IOException {
+ if (writer.hasUncommittedChanges()) {
+ writer.commit();
+ }
+ }
+
+ public void optimize() throws IOException {
+ writer.forceMerge(1);
+ commit();
+ }
+
+ @Override
+ public void close() throws IOException {
+ refreshService.stopAsync().awaitTerminated();
+ commitService.stopAsync().awaitTerminated();
+ writer.close();
+ if (realtimeReader != null) realtimeReader.close();
+ writer.getDirectory().close();
+ }
+
+ public boolean isOpen() {
+ return writer.isOpen();
+ }
+
+ public void addOrReplace(String id, LuceneDocument document) throws IOException {
+ Term uniqueKey = new Term(idField, id);
+ writer.updateDocument(uniqueKey, document.toDocument());
+ }
+
+ public void remove(String id) throws IOException {
+ Term uniqueKey = new Term(idField, id);
+ writer.deleteDocuments(uniqueKey);
+ }
+
+ private class CommitService extends AbstractScheduledService {
+ private final LuceneClient client;
+ private final boolean refresh;
+ private final long delay;
+
+ public CommitService(LuceneClient client, boolean refresh, long ms) {
+ this.client = client;
+ this.refresh = refresh;
+ this.delay = ms;
+ }
+
+ @Override
+ protected void runOneIteration() throws Exception {
+ if (client.isOpen()) {
+ if (!refresh) {
+ client.commit();
+ } else {
+ client.refresh();
+ }
+ }
+ }
+
+ @Override
+ protected Scheduler scheduler() {
+ return Scheduler.newFixedDelaySchedule(delay, delay, TimeUnit.MILLISECONDS);
+ }
+ }
+
+ public LeafReader reader() throws IOException {
+ return SlowCompositeReaderWrapper.wrap(DirectoryReader.open(writer.getDirectory()));
+ }
+
+ public IndexSearcher newSearcher() throws IOException {
+ return new IndexSearcher(DirectoryReader.open(writer.getDirectory()));
+ }
+
+ public IndexSearcher newRealtimeSearcher() throws IOException {
+ if (realtimeReader == null) refresh();
+ return new IndexSearcher(realtimeReader);
+ }
+
+ public Query newQuery(String queryString) {
+ String qstr = Objects.firstNonNull(queryString, "*:*");
+ Query query = null;
+ try {
+ query = queryParser.parse(qstr, contentField);
+ } catch (QueryNodeException e) {
+ query = new MatchAllDocsQuery();
+ }
+ return query;
+ }
+
+ public static String defaultPath() {
+ String sep = StandardSystemProperty.FILE_SEPARATOR.value();
+ String userDir = StandardSystemProperty.USER_DIR.value();
+ return userDir+sep+"lucene"+sep+"data"+sep+"index";
+ }
+
+ public static String defaultCharfilters() {
+ String charfilters =
+ "{" + "\n"
+ + " \"my_htmlstrip\":{\""+ATTR_TYPE+"\":\"htmlstrip\"}" + "\n"
+ + "}";
+ return charfilters;
+ }
+
+ public static String defaultTokenizers() {
+ String tokenizers =
+ "{" + "\n"
+ + " \"my_standard\":{\""+ATTR_TYPE+"\":\"standard\"}," + "\n"
+ + " \"my_whitespace\":{\""+ATTR_TYPE+"\":\"whitespace\"}," + "\n"
+ + " \"my_ngram\":{\""+ATTR_TYPE+"\":\"ngram\",\""+ATTR_PARAMS+"\":{\"minGramSize\":\"2\", \"maxGramSize\":\"2\"}}" + "\n"
+ + "}";
+ return tokenizers;
+ }
+
+ public static String defaultFilters() {
+ String filters =
+ "{" + "\n"
+ + " \"my_stop\":{\""+ATTR_TYPE+"\":\"stop\",\""+ATTR_PARAMS+"\":{\"ignoreCase\":\"true\"}}," + "\n"
+ + " \"my_lowercase\":{\""+ATTR_TYPE+"\":\"lowercase\"}" + "\n"
+ + "}";
+ return filters;
+ }
+
+ public static String defaultAnalyzers() {
+ String analyzers =
+ "{" + "\n"
+ + " \"text_general\":{\""+ATTR_CHARFILTER+"\":[\"my_htmlstrip\"], \""+ATTR_TOKENIZER+"\":\"my_standard\",\""+ATTR_FILTER+"\":[\"my_stop\",\"my_lowercase\"]},"+ "\n"
+ + " \"text_ws\":{\""+ATTR_TOKENIZER+"\":\"my_whitespace\"},"+ "\n"
+ + " \"text_ngram\":{\""+ATTR_TOKENIZER+"\":\"my_ngram\",\""+ATTR_FILTER+"\":[\"my_lowercase\"]}"+ "\n"
+ + "}";
+ return analyzers;
+ }
+
+ public static String defaultFields() {
+ String fields =
+ "{" + "\n"
+ + " \"id\":{\""+ATTR_FIELDTYPE+"\":\""+FIELDTYPE_STRING+"\", \""+ATTR_STORE+"\":true},"+ "\n"
+ + " \"cat\":{\""+ATTR_FIELDTYPE+"\":\""+FIELDTYPE_STRING+"\", \""+ATTR_STORE+"\":true},"+ "\n"
+ + " \"author\":{\""+ATTR_FIELDTYPE+"\":\""+FIELDTYPE_STRING+"\", \""+ATTR_STORE+"\":true},"+ "\n"
+ + " \"content\":{\""+ATTR_FIELDTYPE+"\":\""+FIELDTYPE_TEXT+"\", \""+ATTR_STORE+"\":true,\""+ATTR_INDEX_ANALYZER+"\":\"text_general\",\""+ATTR_QUERY_ANALYZER+"\":\"text_general\",\""+ATTR_COPY_TO+"\":[\"content_ws\", \"content_ngram\"]}," + "\n"
+ + " \"content_ws\":{\""+ATTR_FIELDTYPE+"\":\""+FIELDTYPE_TEXT+"\", \""+ATTR_STORE+"\":false,\""+ATTR_INDEX_ANALYZER+"\":\"text_ws\",\""+ATTR_QUERY_ANALYZER+"\":\"text_ws\"}," + "\n"
+ + " \"content_ngram\":{\""+ATTR_FIELDTYPE+"\":\""+FIELDTYPE_TEXT+"\", \""+ATTR_STORE+"\":false,\""+ATTR_INDEX_ANALYZER+"\":\"text_ngram\",\""+ATTR_QUERY_ANALYZER+"\":\"text_ngram\"}" + "\n"
+ + "}";
+ return fields;
+ }
+
+ public static String defaultIdField() {
+ return "id";
+ }
+
+ public static String defaultContentField() {
+ return "content";
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClient.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClient.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClientManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClientManager.java?rev=1689113&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClientManager.java (added)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClientManager.java Sat Jul 4 07:34:47 2015
@@ -0,0 +1,63 @@
+package org.apache.manifoldcf.agents.output.lucene;
+
+import java.io.File;
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+
+public class LuceneClientManager {
+
+ private static Map<String,LuceneClient> clients = Maps.newHashMap();
+ private static Map<String,String> versionStrings = Maps.newHashMap();
+
+ private LuceneClientManager() { }
+
+ public synchronized static LuceneClient getClient(
+ String path,
+ String charfilters, String tokenizers, String filters,
+ String analyzers, String fields,
+ String idField, String contentField) throws Exception
+ {
+ LuceneClient client = clients.get(path);
+
+ if (client == null) {
+ return newClient(path, charfilters, tokenizers, filters, analyzers, fields, idField, contentField);
+ }
+
+ if (client != null) {
+ if (!client.isOpen()) {
+ return newClient(path, charfilters, tokenizers, filters, analyzers, fields, idField, contentField);
+ }
+ String latestVersion = LuceneClient.createVersionString(
+ new File(path).toPath(),
+ LuceneClient.parseAsMap(charfilters),
+ LuceneClient.parseAsMap(tokenizers),
+ LuceneClient.parseAsMap(filters),
+ LuceneClient.parseAsMap(analyzers),
+ LuceneClient.parseAsMap(fields),
+ idField, contentField);
+ String activeVersion = versionStrings.get(path);
+ if (!activeVersion.equals(latestVersion)) {
+ throw new IllegalStateException("The connection on this path is active. Can not update to the latest settings."
+ + " Active settings:" + activeVersion
+ + " Latest settings:" + latestVersion);
+ }
+ }
+ return client;
+ }
+
+ private static LuceneClient newClient(
+ String path,
+ String charfilters, String tokenizers, String filters,
+ String analyzers, String fields,
+ String idField, String contentField) throws Exception
+ {
+ LuceneClient client = new LuceneClient(new File(path).toPath(),
+ charfilters, tokenizers, filters, analyzers, fields,
+ idField, contentField);
+ clients.put(path, client);
+ versionStrings.put(path, client.versionString());
+ return client;
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClientManager.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClientManager.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConfig.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConfig.java?rev=1689113&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConfig.java (added)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConfig.java Sat Jul 4 07:34:47 2015
@@ -0,0 +1,31 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.agents.output.lucene;
+
+public class LuceneConfig
+{
+ public static final String PARAM_PATH = "path";
+
+ public static final String PARAM_CHARFILTERS = "charfilters";
+ public static final String PARAM_TOKENIZERS = "tokenizers";
+ public static final String PARAM_FILTERS = "filters";
+ public static final String PARAM_ANALYZERS = "analyzers";
+ public static final String PARAM_FIELDS = "fields";
+
+ public static final String PARAM_IDFIELD = "idfield";
+ public static final String PARAM_CONTENTFIELD = "contentfield";
+}
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConfig.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConfig.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConnector.java?rev=1689113&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConnector.java (added)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConnector.java Sat Jul 4 07:34:47 2015
@@ -0,0 +1,571 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.agents.output.lucene;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
+import org.apache.manifoldcf.agents.interfaces.IOutputNotifyActivity;
+import org.apache.manifoldcf.agents.interfaces.IOutputRemoveActivity;
+import org.apache.manifoldcf.agents.interfaces.IOutputCheckActivity;
+import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
+import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
+import org.apache.manifoldcf.core.interfaces.Specification;
+import org.apache.manifoldcf.core.interfaces.ConfigParams;
+import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
+import org.apache.manifoldcf.core.interfaces.IPostParameters;
+import org.apache.manifoldcf.core.interfaces.IThreadContext;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.core.interfaces.VersionContext;
+import org.apache.manifoldcf.crawler.system.Logging;
+
+
+/** This is the output connector for Lucene.
+*/
+public class LuceneConnector extends org.apache.manifoldcf.agents.output.BaseOutputConnector
+{
+ private final static String LUCENE_TAB_PARAMETERS = "LuceneConnector.Parameters";
+
+ /** Forward to the javascript to check the configuration parameters */
+ private static final String EDIT_CONFIG_HEADER_FORWARD = "editConfiguration.js";
+
+ /** Forward to the HTML template to edit the configuration parameters */
+ private static final String EDIT_CONFIG_FORWARD_PARAMETERS = "editConfiguration_Parameters.html";
+
+ /** Forward to the HTML template to view the configuration parameters */
+ private static final String VIEW_CONFIG_FORWARD = "viewConfiguration.html";
+
+ /** Ingestion activity */
+ public final static String INGEST_ACTIVITY = "document ingest";
+ /** Document removal activity */
+ public final static String REMOVE_ACTIVITY = "document deletion";
+
+ /** Connection */
+ private LuceneClient client = null;
+
+ /** Expiration */
+ protected long expirationTime = -1L;
+
+ /** Idle connection expiration interval */
+ protected final static long EXPIRATION_INTERVAL = 300000L;
+
+ public LuceneConnector()
+ {
+ }
+
+ /**
+ * Return the list of activities that this connector supports (i.e. writes
+ * into the log).
+ *
+ * @return the list.
+ */
+ @Override
+ public String[] getActivitiesList()
+ {
+ return new String[]{INGEST_ACTIVITY,REMOVE_ACTIVITY};
+ }
+
+ /** Connect.
+ */
+ @Override
+ public void connect(ConfigParams configParameters)
+ {
+ super.connect(configParameters);
+ }
+
+ /**
+ * This method is called to assess whether to count this connector instance
+ * should actually be counted as being connected.
+ *
+ * @return true if the connector instance is actually connected.
+ */
+ @Override
+ public boolean isConnected()
+ {
+ return client != null;
+ }
+
+ /** Close the connection. Call this before discarding the connection.
+ */
+ @Override
+ public void disconnect() throws ManifoldCFException
+ {
+ if (client != null)
+ {
+ try
+ {
+ client.close();
+ } catch (IOException e) {
+ Logging.connectors.error("disconnect fails:", e);
+ }
+ client = null;
+ expirationTime = -1L;
+ }
+ super.disconnect();
+ }
+
+ protected void getSession() throws ManifoldCFException
+ {
+ if (client == null)
+ {
+ final String path = params.getParameter(LuceneConfig.PARAM_PATH);
+ if (path == null)
+ throw new ManifoldCFException("path not configured");
+
+ final String charfilters = params.getParameter(LuceneConfig.PARAM_CHARFILTERS);
+ if (charfilters == null)
+ throw new ManifoldCFException("charfilters not configured");
+
+ final String tokenizers = params.getParameter(LuceneConfig.PARAM_TOKENIZERS);
+ if (tokenizers == null)
+ throw new ManifoldCFException("tokenizers not configured");
+
+ final String filters = params.getParameter(LuceneConfig.PARAM_FILTERS);
+ if (filters == null)
+ throw new ManifoldCFException("filters not configured");
+
+ final String analyzers = params.getParameter(LuceneConfig.PARAM_ANALYZERS);
+ if (analyzers == null)
+ throw new ManifoldCFException("analyzers not configured");
+
+ final String fields = params.getParameter(LuceneConfig.PARAM_FIELDS);
+ if (fields == null)
+ throw new ManifoldCFException("fields not configured");
+
+ final String idField = params.getParameter(LuceneConfig.PARAM_IDFIELD);
+ if (idField == null)
+ throw new ManifoldCFException("id field not configured");
+
+ final String contentField = params.getParameter(LuceneConfig.PARAM_CONTENTFIELD);
+ if (contentField == null)
+ throw new ManifoldCFException("content field not configured");
+
+ try
+ {
+ client = LuceneClientManager.getClient(path,
+ charfilters, tokenizers, filters, analyzers, fields,
+ idField, contentField);
+ }
+ catch (Exception e)
+ {
+ throw new ManifoldCFException(e);
+ }
+ }
+ expirationTime = System.currentTimeMillis() + EXPIRATION_INTERVAL;
+ }
+
+ @Override
+ public String check() throws ManifoldCFException
+ {
+ try
+ {
+ getSession();
+ } catch (ManifoldCFException e) {
+ Logging.connectors.error("Connection Not Working", e);
+ return "Connection Not Working! " + e.getMessage();
+ }
+ return super.check();
+ }
+
+ /** This method is periodically called for all connectors that are connected but not
+ * in active use.
+ */
+ @Override
+ public void poll() throws ManifoldCFException
+ {
+ if (client != null)
+ {
+ if (expirationTime <= System.currentTimeMillis())
+ {
+ try
+ {
+ client.close();
+ } catch (IOException e) {
+ Logging.connectors.error("poll fails:", e);
+ }
+ client = null;
+ expirationTime = -1L;
+ }
+ }
+ }
+
+ /**
+ * Get an output version string, given an output specification. The output
+ * version string is used to uniquely describe the pertinent details of the
+ * output specification and the configuration, to allow the Connector
+ * Framework to determine whether a document will need to be output again.
+ * Note that the contents of the document cannot be considered by this
+ * method, and that a different version string (defined in
+ * IRepositoryConnector) is used to describe the version of the actual
+ * document.
+ * <p/>
+ * This method presumes that the connector object has been configured, and
+ * it is thus able to communicate with the output data store should that be
+ * necessary.
+ *
+ * @param spec
+ * is the current output specification for the job that is doing
+ * the crawling.
+ * @return a string, of unlimited length, which uniquely describes output
+ * configuration and specification in such a way that if two such
+ * strings are equal, the document will not need to be sent again to
+ * the output data store.
+ */
+ @Override
+ public VersionContext getPipelineDescription(Specification spec)
+ throws ManifoldCFException, ServiceInterruption {
+ getSession();
+ return new VersionContext(client.versionString(), params, spec);
+ }
+
+ /**
+ * Detect if a mime type is indexable or not. This method is used by
+ * participating repository connectors to pre-filter the number of unusable
+ * documents that will be passed to this output connector.
+ *
+ * @param outputDescription
+ * is the document's output version.
+ * @param mimeType
+ * is the mime type of the document.
+ * @return true if the mime type is indexable by this connector.
+ */
+ @Override
+ public boolean checkMimeTypeIndexable(VersionContext outputDescription,
+ String mimeType, IOutputCheckActivity activities)
+ throws ManifoldCFException, ServiceInterruption {
+ return true;
+ }
+
+ /**
+ * Pre-determine whether a document's length is indexable by this connector.
+ * This method is used by participating repository connectors to help filter
+ * out documents that are too long to be indexable.
+ *
+ * @param outputDescription
+ * is the document's output version.
+ * @param length
+ * is the length of the document.
+ * @return true if the file is indexable.
+ */
+ @Override
+ public boolean checkLengthIndexable(VersionContext outputDescription,
+ long length, IOutputCheckActivity activities)
+ throws ManifoldCFException, ServiceInterruption {
+ return true;
+ }
+
+ /**
+ * Add (or replace) a document in the output data store using the connector.
+ * This method presumes that the connector object has been configured, and
+ * it is thus able to communicate with the output data store should that be
+ * necessary.
+ *
+ * @param documentURI
+ * is the URI of the document. The URI is presumed to be the
+ * unique identifier which the output data store will use to
+ * process and serve the document. This URI is constructed by the
+ * repository connector which fetches the document, and is thus
+ * universal across all output connectors.
+ * @param pipelineDescription
+ * includes the description string that was constructed for this
+ * document by the getOutputDescription() method.
+ * @param document
+ * is the document data to be processed (handed to the output
+ * data store).
+ * @param authorityNameString
+ * is the name of the authority responsible for authorizing any
+ * access tokens passed in with the repository document. May be
+ * null.
+ * @param activities
+ * is the handle to an object that the implementer of a pipeline
+ * connector may use to perform operations, such as logging
+ * processing activity, or sending a modified document to the
+ * next stage in the pipeline.
+ * @return the document status (accepted or permanently rejected).
+ * @throws IOException
+ * only if there's a stream error reading the document data.
+ */
+ @Override
+ public int addOrReplaceDocumentWithException(String documentURI, VersionContext pipelineDescription, RepositoryDocument document, String authorityNameString, IOutputAddActivity activities)
+ throws ManifoldCFException, ServiceInterruption, IOException
+ {
+ getSession();
+
+ LuceneDocument inputDoc = buildDocument(documentURI, document);
+
+ long startTime = System.currentTimeMillis();
+ try
+ {
+ client.addOrReplace(documentURI, inputDoc);
+ activities.recordActivity(startTime, INGEST_ACTIVITY, null, documentURI, "OK", "Document Indexed");
+ } catch (IOException e) {
+ String activityCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ String activityDetails = e.getMessage() + ((e.getCause() != null) ? ": "+ e.getCause().getMessage() : "");
+ activities.recordActivity(startTime, INGEST_ACTIVITY, null, documentURI, activityCode, activityDetails);
+ return DOCUMENTSTATUS_REJECTED;
+ }
+ return DOCUMENTSTATUS_ACCEPTED;
+ }
+
+ private LuceneDocument buildDocument(String documentURI, RepositoryDocument document) {
+ LuceneDocument doc = new LuceneDocument();
+
+ doc = LuceneDocument.addField(doc, client.idField(), documentURI, client.fieldsInfo());
+
+ try
+ {
+ StringWriter writer = new StringWriter();
+ IOUtils.copy(document.getBinaryStream(), writer, StandardCharsets.UTF_8);
+ doc = LuceneDocument.addField(doc, client.contentField(), writer.toString(), client.fieldsInfo());
+ } catch (IOException e) {
+ Logging.connectors.error("[Parsing Content]Content is not text plain, verify you are properly using Apache Tika Transformer", e);
+ }
+
+ Iterator<String> it = document.getFields();
+ while (it.hasNext()) {
+ String rdField = it.next();
+ if (client.fieldsInfo().containsKey(rdField)) {
+ try
+ {
+ String[] values = document.getFieldAsStrings(rdField);
+ for (String value : values) {
+ doc = LuceneDocument.addField(doc, rdField, value, client.fieldsInfo());
+ }
+ } catch (IOException e) {
+ Logging.connectors.error("[Getting Field Values]Impossible to read value for metadata " + rdField, e);
+ }
+ }
+ }
+ return doc;
+ }
+
+ /**
+ * Remove a document using the connector. Note that the last
+ * outputDescription is included, since it may be necessary for the
+ * connector to use such information to know how to properly remove the
+ * document.
+ *
+ * @param documentURI
+ * is the URI of the document. The URI is presumed to be the
+ * unique identifier which the output data store will use to
+ * process and serve the document. This URI is constructed by the
+ * repository connector which fetches the document, and is thus
+ * universal across all output connectors.
+ * @param outputDescription
+ * is the last description string that was constructed for this
+ * document by the getOutputDescription() method above.
+ * @param activities
+ * is the handle to an object that the implementer of an output
+ * connector may use to perform operations, such as logging
+ * processing activity.
+ */
+ @Override
+ public void removeDocument(String documentURI, String outputDescription, IOutputRemoveActivity activities)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ getSession();
+
+ long startTime = System.currentTimeMillis();
+ try
+ {
+ client.remove(documentURI);
+ activities.recordActivity(startTime, REMOVE_ACTIVITY, null, documentURI, "OK", "Document Deleted");
+ } catch (IOException e) {
+ String activityCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT);
+ String activityDetails = e.getMessage() + ((e.getCause() != null) ? ": "+ e.getCause().getMessage() : "");
+ activities.recordActivity(startTime, REMOVE_ACTIVITY, null, documentURI, activityCode, activityDetails);
+ }
+ }
+
+ /** Notify the connector of a completed job.
+ * This is meant to allow the connector to flush any internal data structures it has been keeping around, or to tell the output repository that this
+ * is a good time to synchronize things. It is called whenever a job is either completed or aborted.
+ *@param activities is the handle to an object that the implementer of an output connector may use to perform operations, such as logging processing activity.
+ */
+ @Override
+ public void noteJobComplete(IOutputNotifyActivity activities)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ getSession();
+ try
+ {
+ client.optimize();
+ } catch (IOException e) {
+ Logging.connectors.error("noteJobComplete fails:", e);
+ }
+ }
+
+ /**
+ * Read the content of a resource, replace the variable ${PARAMNAME} with
+ * the value and copy it to the out.
+ *
+ * @param resName
+ * @param out
+ * @throws ManifoldCFException
+ */
+ private static void outputResource(String resName, IHTTPOutput out,
+ Locale locale, Map<String, String> params, String tabName,
+ Integer sequenceNumber, Integer currentSequenceNumber)
+ throws ManifoldCFException {
+ Map<String, String> paramMap = null;
+ if (params != null) {
+ paramMap = params;
+ if (tabName != null) {
+ paramMap.put("TabName", tabName);
+ }
+ if (currentSequenceNumber != null)
+ paramMap.put("SelectedNum", currentSequenceNumber.toString());
+ } else {
+ paramMap = new HashMap<String, String>();
+ }
+ if (sequenceNumber != null)
+ paramMap.put("SeqNum", sequenceNumber.toString());
+
+ Messages.outputResourceWithVelocity(out, locale, resName, paramMap, true);
+ }
+
+ @Override
+ public void outputConfigurationHeader(IThreadContext threadContext,
+ IHTTPOutput out, Locale locale, ConfigParams parameters,
+ List<String> tabsArray) throws ManifoldCFException, IOException {
+ super.outputConfigurationHeader(threadContext, out, locale, parameters,
+ tabsArray);
+ tabsArray.add(Messages.getString(locale, LUCENE_TAB_PARAMETERS));
+ outputResource(EDIT_CONFIG_HEADER_FORWARD, out, locale, null, null, null, null);
+ }
+
+ @Override
+ public void outputConfigurationBody(IThreadContext threadContext,
+ IHTTPOutput out, Locale locale, ConfigParams parameters,
+ String tabName) throws ManifoldCFException, IOException {
+ super.outputConfigurationBody(threadContext, out, locale, parameters, tabName);
+ Map<String, String> config = getConfigParameters(parameters);
+ outputResource(EDIT_CONFIG_FORWARD_PARAMETERS, out, locale, config, tabName, null, null);
+ }
+
+ private final Map<String, String> getConfigParameters(ConfigParams configParams) {
+ Map<String, String> map = new HashMap<String, String>();
+
+ String path = configParams.getParameter(LuceneConfig.PARAM_PATH);
+ if (path == null)
+ path = LuceneClient.defaultPath();
+ map.put(LuceneConfig.PARAM_PATH, path);
+
+ String charfilters = configParams.getParameter(LuceneConfig.PARAM_CHARFILTERS);
+ if (charfilters == null)
+ charfilters = LuceneClient.defaultCharfilters();
+ map.put(LuceneConfig.PARAM_CHARFILTERS, charfilters);
+
+ String tokenizers = configParams.getParameter(LuceneConfig.PARAM_TOKENIZERS);
+ if (tokenizers == null)
+ tokenizers = LuceneClient.defaultTokenizers();
+ map.put(LuceneConfig.PARAM_TOKENIZERS, tokenizers);
+
+ String filters = configParams.getParameter(LuceneConfig.PARAM_FILTERS);
+ if (filters == null)
+ filters = LuceneClient.defaultFilters();
+ map.put(LuceneConfig.PARAM_FILTERS, filters);
+
+ String analyzers = configParams.getParameter(LuceneConfig.PARAM_ANALYZERS);
+ if (analyzers == null)
+ analyzers = LuceneClient.defaultAnalyzers();
+ map.put(LuceneConfig.PARAM_ANALYZERS, analyzers);
+
+ String fields = configParams.getParameter(LuceneConfig.PARAM_FIELDS);
+ if (fields == null)
+ fields = LuceneClient.defaultFields();
+ map.put(LuceneConfig.PARAM_FIELDS, fields);
+
+ String idField = configParams.getParameter(LuceneConfig.PARAM_IDFIELD);
+ if (idField == null)
+ idField = LuceneClient.defaultIdField();
+ map.put(LuceneConfig.PARAM_IDFIELD, idField);
+
+ String contentField = configParams.getParameter(LuceneConfig.PARAM_CONTENTFIELD);
+ if (contentField == null)
+ contentField = LuceneClient.defaultContentField();
+ map.put(LuceneConfig.PARAM_CONTENTFIELD, contentField);
+
+ return map;
+ }
+
+ @Override
+ public void viewConfiguration(IThreadContext threadContext,
+ IHTTPOutput out, Locale locale, ConfigParams parameters)
+ throws ManifoldCFException, IOException {
+ outputResource(VIEW_CONFIG_FORWARD, out, locale, getConfigParameters(parameters), null, null, null);
+ }
+
+ /**
+ * Process a configuration post. This method is called at the start of the
+ * connector's configuration page, whenever there is a possibility that form
+ * data for a connection has been posted. Its purpose is to gather form
+ * information and modify the configuration parameters accordingly. The name
+ * of the posted form is "editconnection".
+ *
+ * @param threadContext
+ * is the local thread context.
+ * @param variableContext
+ * is the set of variables available from the post, including
+ * binary file post information.
+ * @param parameters
+ * are the configuration parameters, as they currently exist, for
+ * this connection being configured.
+ * @return null if all is well, or a string error message if there is an
+ * error that should prevent saving of the connection (and cause a
+ * redirection to an error page).
+ */
+ @Override
+ public String processConfigurationPost(IThreadContext threadContext,
+ IPostParameters variableContext, Locale locale,
+ ConfigParams parameters) throws ManifoldCFException {
+ String path = variableContext.getParameter(LuceneConfig.PARAM_PATH);
+ if (path != null)
+ parameters.setParameter(LuceneConfig.PARAM_PATH, path);
+ String charfilters = variableContext.getParameter(LuceneConfig.PARAM_CHARFILTERS);
+ if (charfilters != null)
+ parameters.setParameter(LuceneConfig.PARAM_CHARFILTERS, charfilters);
+ String tokenizers = variableContext.getParameter(LuceneConfig.PARAM_TOKENIZERS);
+ if (tokenizers != null)
+ parameters.setParameter(LuceneConfig.PARAM_TOKENIZERS, tokenizers);
+ String filters = variableContext.getParameter(LuceneConfig.PARAM_FILTERS);
+ if (filters != null)
+ parameters.setParameter(LuceneConfig.PARAM_FILTERS, filters);
+ String analyzers = variableContext.getParameter(LuceneConfig.PARAM_ANALYZERS);
+ if (analyzers != null)
+ parameters.setParameter(LuceneConfig.PARAM_ANALYZERS, analyzers);
+ String fields = variableContext.getParameter(LuceneConfig.PARAM_FIELDS);
+ if (fields != null)
+ parameters.setParameter(LuceneConfig.PARAM_FIELDS, fields);
+ String idFields = variableContext.getParameter(LuceneConfig.PARAM_IDFIELD);
+ if (idFields != null)
+ parameters.setParameter(LuceneConfig.PARAM_IDFIELD, idFields);
+ String contentFields = variableContext.getParameter(LuceneConfig.PARAM_CONTENTFIELD);
+ if (contentFields != null)
+ parameters.setParameter(LuceneConfig.PARAM_CONTENTFIELD, contentFields);
+ return null;
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConnector.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConnector.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneDocument.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneDocument.java?rev=1689113&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneDocument.java (added)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneDocument.java Sat Jul 4 07:34:47 2015
@@ -0,0 +1,89 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.agents.output.lucene;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import com.google.common.base.Objects;
+
+public class LuceneDocument {
+
+ private Document doc;
+
+ private static final FieldType TYPE_STORED_WITH_TV = new FieldType(TextField.TYPE_STORED);
+ static {
+ TYPE_STORED_WITH_TV.setStoreTermVectors(true);
+ TYPE_STORED_WITH_TV.setStoreTermVectorOffsets(true);
+ TYPE_STORED_WITH_TV.setStoreTermVectorPositions(true);
+ TYPE_STORED_WITH_TV.freeze();
+ }
+
+ private static final FieldType TYPE_NOT_STORED_WITH_TV = new FieldType(TextField.TYPE_NOT_STORED);
+ static {
+ TYPE_NOT_STORED_WITH_TV.setStoreTermVectors(true);
+ TYPE_NOT_STORED_WITH_TV.setStoreTermVectorOffsets(true);
+ TYPE_NOT_STORED_WITH_TV.setStoreTermVectorPositions(true);
+ TYPE_NOT_STORED_WITH_TV.freeze();
+ }
+
+ public LuceneDocument() {
+ doc = new Document();
+ }
+
+ public LuceneDocument addStringField(String name, String value, boolean store) {
+ Store stored = (store) ? Field.Store.YES : Field.Store.NO;
+ doc.add(new StringField(name, value, stored));
+ return this;
+ }
+
+ public LuceneDocument addTextField(String name, String value, boolean store) {
+ FieldType type = (store) ? TYPE_STORED_WITH_TV : TYPE_NOT_STORED_WITH_TV;
+ doc.add(new Field(name, value, type));
+ return this;
+ }
+
+ public Document toDocument() {
+ return doc;
+ }
+
+ public static LuceneDocument addField(LuceneDocument from, String field, String value, Map<String,Map<String,Object>> fieldsInfo) {
+ String fieldtype = (String)fieldsInfo.get(field).get(LuceneClient.ATTR_FIELDTYPE);
+ boolean store = (boolean)Objects.firstNonNull(fieldsInfo.get(field).get(LuceneClient.ATTR_STORE), false);
+
+ if (fieldtype.equals(LuceneClient.FIELDTYPE_TEXT)) {
+ from.addTextField(field, value, store);
+ } else if (fieldtype.equals(LuceneClient.FIELDTYPE_STRING)) {
+ from.addStringField(field, value, store);
+ }
+
+ @SuppressWarnings("unchecked")
+ List<String> copyFields = (List<String>)Objects.firstNonNull(fieldsInfo.get(field).get(LuceneClient.ATTR_COPY_TO), new ArrayList<String>());
+ for (String tofield : copyFields) {
+ from = addField(from, tofield, value, fieldsInfo);
+ }
+ return from;
+ }
+
+}
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneDocument.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneDocument.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/Messages.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/Messages.java?rev=1689113&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/Messages.java (added)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/Messages.java Sat Jul 4 07:34:47 2015
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.manifoldcf.agents.output.lucene;
+
+import java.util.Locale;
+import java.util.Map;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
+
+public class Messages extends org.apache.manifoldcf.ui.i18n.Messages
+{
+ public static final String DEFAULT_BUNDLE_NAME = "org.apache.manifoldcf.agents.output.lucene.common";
+ public static final String DEFAULT_PATH_NAME = "org.apache.manifoldcf.agents.output.lucene";
+
+ /** Constructor - do no instantiate
+ */
+ protected Messages()
+ {
+ }
+
+ public static String getString(Locale locale, String messageKey)
+ {
+ return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getAttributeString(Locale locale, String messageKey)
+ {
+ return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getBodyString(Locale locale, String messageKey)
+ {
+ return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getAttributeJavascriptString(Locale locale, String messageKey)
+ {
+ return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getBodyJavascriptString(Locale locale, String messageKey)
+ {
+ return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, null);
+ }
+
+ public static String getString(Locale locale, String messageKey, Object[] args)
+ {
+ return getString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getAttributeString(Locale locale, String messageKey, Object[] args)
+ {
+ return getAttributeString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getBodyString(Locale locale, String messageKey, Object[] args)
+ {
+ return getBodyString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getAttributeJavascriptString(Locale locale, String messageKey, Object[] args)
+ {
+ return getAttributeJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ public static String getBodyJavascriptString(Locale locale, String messageKey, Object[] args)
+ {
+ return getBodyJavascriptString(DEFAULT_BUNDLE_NAME, locale, messageKey, args);
+ }
+
+ // More general methods which allow bundlenames and class loaders to be specified.
+
+ public static String getString(String bundleName, Locale locale, String messageKey, Object[] args)
+ {
+ return getString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ public static String getAttributeString(String bundleName, Locale locale, String messageKey, Object[] args)
+ {
+ return getAttributeString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ public static String getBodyString(String bundleName, Locale locale, String messageKey, Object[] args)
+ {
+ return getBodyString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ public static String getAttributeJavascriptString(String bundleName, Locale locale, String messageKey, Object[] args)
+ {
+ return getAttributeJavascriptString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ public static String getBodyJavascriptString(String bundleName, Locale locale, String messageKey, Object[] args)
+ {
+ return getBodyJavascriptString(Messages.class, bundleName, locale, messageKey, args);
+ }
+
+ // Resource output
+
+ public static void outputResource(IHTTPOutput output, Locale locale, String resourceKey,
+ Map<String,String> substitutionParameters, boolean mapToUpperCase)
+ throws ManifoldCFException
+ {
+ outputResource(output,Messages.class,DEFAULT_PATH_NAME,locale,resourceKey,
+ substitutionParameters,mapToUpperCase);
+ }
+
+ public static void outputResourceWithVelocity(IHTTPOutput output, Locale locale, String resourceKey,
+ Map<String,String> substitutionParameters, boolean mapToUpperCase)
+ throws ManifoldCFException
+ {
+ outputResourceWithVelocity(output,Messages.class,DEFAULT_BUNDLE_NAME,DEFAULT_PATH_NAME,locale,resourceKey,
+ substitutionParameters,mapToUpperCase);
+ }
+
+ public static void outputResourceWithVelocity(IHTTPOutput output, Locale locale, String resourceKey,
+ Map<String,Object> contextObjects)
+ throws ManifoldCFException
+ {
+ outputResourceWithVelocity(output,Messages.class,DEFAULT_BUNDLE_NAME,DEFAULT_PATH_NAME,locale,resourceKey,
+ contextObjects);
+ }
+
+}
+
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/Messages.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/Messages.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_en_US.properties?rev=1689113&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_en_US.properties (added)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_en_US.properties Sat Jul 4 07:34:47 2015
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LuceneConnector.Parameters=Lucene Settings
+LuceneConnector.Configuration=Lucene
+
+
+LuceneConnector.Path=Path
+LuceneConnector.Charfilters=Charfilters
+LuceneConnector.Tokenizers=Tokenizers
+LuceneConnector.Filters=Filters
+LuceneConnector.Analyzers=Analyzers
+LuceneConnector.Fields=Fields
+LuceneConnector.Idfield=id field name
+LuceneConnector.Contentfield=content field name
+
+
+LuceneConnector.PleaseSupplyValidPath=Path can't be empty. Please supply a valid path
+LuceneConnector.PleaseSupplyValidCharfilters=Charfilters can't be empty. Please supply a valid charfilters
+LuceneConnector.PleaseSupplyValidTokenizers=Tokenizers can't be empty. Please supply a valid tokenizers
+LuceneConnector.PleaseSupplyValidFilters=Filters can't be empty. Please supply a valid filters
+LuceneConnector.PleaseSupplyValidAnalyzers=Analyzers can't be empty. Please supply a valid analyzers
+LuceneConnector.PleaseSupplyValidFields=Fields can't be empty. Please supply a valid fields
+LuceneConnector.PleaseSupplyValidIdfield=Idfield can't be empty. Please supply a valid idfield
+LuceneConnector.PleaseSupplyValidContentfield=Contentfield can't be empty. Please supply a valid contentfield
+
+
+
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_en_US.properties
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_en_US.properties
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_ja_JP.properties?rev=1689113&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_ja_JP.properties (added)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_ja_JP.properties Sat Jul 4 07:34:47 2015
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LuceneConnector.Parameters=Lucene Settings
+LuceneConnector.Configuration=Lucene
+
+
+LuceneConnector.Path=Path
+LuceneConnector.Charfilters=Charfilters
+LuceneConnector.Tokenizers=Tokenizers
+LuceneConnector.Filters=Filters
+LuceneConnector.Analyzers=Analyzers
+LuceneConnector.Fields=Fields
+LuceneConnector.Idfield=id field name
+LuceneConnector.Contentfield=content field name
+
+
+LuceneConnector.PleaseSupplyValidPath=Path can't be empty. Please supply a valid path
+LuceneConnector.PleaseSupplyValidCharfilters=Charfilters can't be empty. Please supply a valid charfilters
+LuceneConnector.PleaseSupplyValidTokenizers=Tokenizers can't be empty. Please supply a valid tokenizers
+LuceneConnector.PleaseSupplyValidFilters=Filters can't be empty. Please supply a valid filters
+LuceneConnector.PleaseSupplyValidAnalyzers=Analyzers can't be empty. Please supply a valid analyzers
+LuceneConnector.PleaseSupplyValidFields=Fields can't be empty. Please supply a valid fields
+LuceneConnector.PleaseSupplyValidIdfield=Idfield can't be empty. Please supply a valid idfield
+LuceneConnector.PleaseSupplyValidContentfield=Contentfield can't be empty. Please supply a valid contentfield
+
+
+
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_ja_JP.properties
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_ja_JP.properties
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_zh_CN.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_zh_CN.properties?rev=1689113&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_zh_CN.properties (added)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_zh_CN.properties Sat Jul 4 07:34:47 2015
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LuceneConnector.Parameters=Lucene Settings
+LuceneConnector.Configuration=Lucene
+
+
+LuceneConnector.Path=Path
+LuceneConnector.Charfilters=Charfilters
+LuceneConnector.Tokenizers=Tokenizers
+LuceneConnector.Filters=Filters
+LuceneConnector.Analyzers=Analyzers
+LuceneConnector.Fields=Fields
+LuceneConnector.Idfield=id field name
+LuceneConnector.Contentfield=content field name
+
+
+LuceneConnector.PleaseSupplyValidPath=Path can't be empty. Please supply a valid path
+LuceneConnector.PleaseSupplyValidCharfilters=Charfilters can't be empty. Please supply a valid charfilters
+LuceneConnector.PleaseSupplyValidTokenizers=Tokenizers can't be empty. Please supply a valid tokenizers
+LuceneConnector.PleaseSupplyValidFilters=Filters can't be empty. Please supply a valid filters
+LuceneConnector.PleaseSupplyValidAnalyzers=Analyzers can't be empty. Please supply a valid analyzers
+LuceneConnector.PleaseSupplyValidFields=Fields can't be empty. Please supply a valid fields
+LuceneConnector.PleaseSupplyValidIdfield=Idfield can't be empty. Please supply a valid idfield
+LuceneConnector.PleaseSupplyValidContentfield=Contentfield can't be empty. Please supply a valid contentfield
+
+
+
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_zh_CN.properties
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_zh_CN.properties
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL