You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by sh...@apache.org on 2015/07/06 21:02:37 UTC
svn commit: r1689479 - in
/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src:
main/java/org/apache/manifoldcf/agents/output/lucene/
main/native2ascii/org/apache/manifoldcf/agents/output/lucene/
main/resources/org/apache/manifoldcf/age...
Author: shinichiro
Date: Mon Jul 6 19:02:37 2015
New Revision: 1689479
URL: http://svn.apache.org/r1689479
Log:
Added Maximumdocumentlength params and field
Modified:
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClient.java
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClientManager.java
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConfig.java
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConnector.java
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneDocument.java
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_en_US.properties
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_ja_JP.properties
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_zh_CN.properties
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/lucene/editConfiguration.js
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/lucene/editConfiguration_Parameters.html
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/lucene/viewConfiguration.html
manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/test/java/org/apache/manifoldcf/agents/output/lucene/tests/LuceneClientTest.java
Modified: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClient.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClient.java?rev=1689479&r1=1689478&r2=1689479&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClient.java (original)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClient.java Mon Jul 6 19:02:37 2015
@@ -67,6 +67,7 @@ public class LuceneClient implements Clo
private final Map<String,Map<String,Object>> fieldsInfo;
private final String idField;
private final String contentField;
+ private final Long maximumDocumentLength;
private final String versionString;
@@ -105,13 +106,15 @@ public class LuceneClient implements Clo
this(path,
LuceneClient.defaultCharfilters(), LuceneClient.defaultTokenizers(), LuceneClient.defaultFilters(),
LuceneClient.defaultAnalyzers(), LuceneClient.defaultFields(),
- LuceneClient.defaultIdField(), LuceneClient.defaultContentField());
+ LuceneClient.defaultIdField(), LuceneClient.defaultContentField(),
+ LuceneClient.defaultMaximumDocumentLength());
}
public LuceneClient(Path path,
String charfilters, String tokenizers, String filters,
String analyzers, String fields,
- String idField, String contentField) throws IOException {
+ String idField, String contentField,
+ Long maximumDocumentLength) throws IOException {
this.path = Preconditions.checkNotNull(path);
this.charfiltersInfo = parseAsMap(Preconditions.checkNotNull(charfilters));
this.tokenizersInfo = parseAsMap(Preconditions.checkNotNull(tokenizers));
@@ -120,8 +123,9 @@ public class LuceneClient implements Clo
this.fieldsInfo = parseAsMap(Preconditions.checkNotNull(fields));
this.idField = Preconditions.checkNotNull(idField);
this.contentField = Preconditions.checkNotNull(contentField);
+ this.maximumDocumentLength = Preconditions.checkNotNull(maximumDocumentLength);
- this.versionString = createVersionString(path, charfiltersInfo, tokenizersInfo, filtersInfo, analyzersInfo, fieldsInfo, idField, contentField);
+ this.versionString = createVersionString(path, charfiltersInfo, tokenizersInfo, filtersInfo, analyzersInfo, fieldsInfo, idField, contentField, maximumDocumentLength);
Map<String,Analyzer> analyzersMap = createAnalyzersMap();
Map<String,Analyzer> fieldIndexAnalyzers = createFieldAnalyzers(analyzersMap, ATTR_INDEX_ANALYZER);
@@ -252,6 +256,10 @@ public class LuceneClient implements Clo
return contentField;
}
+ public Long maximumDocumentLength() {
+ return maximumDocumentLength;
+ }
+
public String versionString() {
return versionString;
}
@@ -263,7 +271,8 @@ public class LuceneClient implements Clo
Map<String,Map<String,Object>> filtersInfo,
Map<String,Map<String,Object>> analyzersInfo,
Map<String,Map<String,Object>> fieldsInfo,
- String idField,String contentField) {
+ String idField,String contentField,
+ Long maximumDocumentLength) {
return LuceneConfig.PARAM_PATH + ":" + path.toString() + "+"
+ LuceneConfig.PARAM_CHARFILTERS + ":" + Joiner.on(",").withKeyValueSeparator("=").join(charfiltersInfo) + "+"
+ LuceneConfig.PARAM_TOKENIZERS + ":" + Joiner.on(",").withKeyValueSeparator("=").join(tokenizersInfo) + "+"
@@ -271,7 +280,8 @@ public class LuceneClient implements Clo
+ LuceneConfig.PARAM_ANALYZERS + ":" + Joiner.on(",").withKeyValueSeparator("=").join(analyzersInfo) + "+"
+ LuceneConfig.PARAM_FIELDS + ":" + Joiner.on(",").withKeyValueSeparator("=").join(fieldsInfo) + "+"
+ LuceneConfig.PARAM_IDFIELD + ":" + idField + "+"
- + LuceneConfig.PARAM_CONTENTFIELD + ":" + contentField;
+ + LuceneConfig.PARAM_CONTENTFIELD + ":" + contentField + "+"
+ + LuceneConfig.PARAM_MAXIMUMDOCUMENTLENGTH + ":" + maximumDocumentLength.toString();
}
public void refresh() throws IOException {
@@ -432,4 +442,8 @@ public class LuceneClient implements Clo
return "content";
}
+ public static Long defaultMaximumDocumentLength() {
+ return new Long(2100000000L);
+ }
+
}
Modified: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClientManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClientManager.java?rev=1689479&r1=1689478&r2=1689479&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClientManager.java (original)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneClientManager.java Mon Jul 6 19:02:37 2015
@@ -15,17 +15,18 @@ public class LuceneClientManager {
String path,
String charfilters, String tokenizers, String filters,
String analyzers, String fields,
- String idField, String contentField) throws Exception
+ String idField, String contentField,
+ Long maximumDocumentLength) throws Exception
{
LuceneClient client = clients.get(path);
if (client == null) {
- return newClient(path, charfilters, tokenizers, filters, analyzers, fields, idField, contentField);
+ return newClient(path, charfilters, tokenizers, filters, analyzers, fields, idField, contentField, maximumDocumentLength);
}
if (client != null) {
if (!client.isOpen()) {
- return newClient(path, charfilters, tokenizers, filters, analyzers, fields, idField, contentField);
+ return newClient(path, charfilters, tokenizers, filters, analyzers, fields, idField, contentField, maximumDocumentLength);
}
String latestVersion = LuceneClient.createVersionString(
new File(path).toPath(),
@@ -34,7 +35,7 @@ public class LuceneClientManager {
LuceneClient.parseAsMap(filters),
LuceneClient.parseAsMap(analyzers),
LuceneClient.parseAsMap(fields),
- idField, contentField);
+ idField, contentField, maximumDocumentLength);
String activeVersion = client.versionString();
if (!activeVersion.equals(latestVersion)) {
throw new IllegalStateException("The connection on this path is active. Can not update to the latest settings."
@@ -49,11 +50,12 @@ public class LuceneClientManager {
String path,
String charfilters, String tokenizers, String filters,
String analyzers, String fields,
- String idField, String contentField) throws Exception
+ String idField, String contentField,
+ Long maximumDocumentLength) throws Exception
{
LuceneClient client = new LuceneClient(new File(path).toPath(),
charfilters, tokenizers, filters, analyzers, fields,
- idField, contentField);
+ idField, contentField, maximumDocumentLength);
clients.put(path, client);
return client;
}
Modified: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConfig.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConfig.java?rev=1689479&r1=1689478&r2=1689479&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConfig.java (original)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConfig.java Mon Jul 6 19:02:37 2015
@@ -28,4 +28,6 @@ public class LuceneConfig
public static final String PARAM_IDFIELD = "idfield";
public static final String PARAM_CONTENTFIELD = "contentfield";
+
+ public static final String PARAM_MAXIMUMDOCUMENTLENGTH = "maximumdocumentlength";
}
Modified: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConnector.java?rev=1689479&r1=1689478&r2=1689479&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneConnector.java Mon Jul 6 19:02:37 2015
@@ -162,11 +162,16 @@ public class LuceneConnector extends org
if (contentField == null)
throw new ManifoldCFException("content field not configured");
+ final String maxDocumentLength = params.getParameter(LuceneConfig.PARAM_MAXIMUMDOCUMENTLENGTH);
+ if (maxDocumentLength == null)
+ throw new ManifoldCFException("maximum document length not configured");
+ Long maximumDocumentLength = new Long(maxDocumentLength);
+
try
{
client = LuceneClientManager.getClient(path,
charfilters, tokenizers, filters, analyzers, fields,
- idField, contentField);
+ idField, contentField, maximumDocumentLength);
}
catch (Exception e)
{
@@ -273,6 +278,9 @@ public class LuceneConnector extends org
public boolean checkLengthIndexable(VersionContext outputDescription,
long length, IOutputCheckActivity activities)
throws ManifoldCFException, ServiceInterruption {
+ getSession();
+ if (length > client.maximumDocumentLength())
+ return false;
return true;
}
@@ -313,6 +321,11 @@ public class LuceneConnector extends org
{
getSession();
+ if (client.maximumDocumentLength() != null && document.getBinaryLength() > client.maximumDocumentLength().longValue()){
+ activities.recordActivity(null, INGEST_ACTIVITY, null, documentURI, activities.EXCLUDED_LENGTH, "Lucene connector rejected document due to its big size: ('"+document.getBinaryLength()+"')");
+ return DOCUMENTSTATUS_REJECTED;
+ }
+
long startTime = System.currentTimeMillis();
try
{
@@ -520,6 +533,11 @@ public class LuceneConnector extends org
contentField = LuceneClient.defaultContentField();
map.put(LuceneConfig.PARAM_CONTENTFIELD, contentField);
+ String maximumDocumentLength = configParams.getParameter(LuceneConfig.PARAM_MAXIMUMDOCUMENTLENGTH);
+ if (maximumDocumentLength == null)
+ maximumDocumentLength = LuceneClient.defaultMaximumDocumentLength().toString();
+ map.put(LuceneConfig.PARAM_MAXIMUMDOCUMENTLENGTH, maximumDocumentLength);
+
return map;
}
@@ -577,6 +595,9 @@ public class LuceneConnector extends org
String contentFields = variableContext.getParameter(LuceneConfig.PARAM_CONTENTFIELD);
if (contentFields != null)
parameters.setParameter(LuceneConfig.PARAM_CONTENTFIELD, contentFields);
+ String maximumDocumentLength = variableContext.getParameter(LuceneConfig.PARAM_MAXIMUMDOCUMENTLENGTH);
+ if (maximumDocumentLength != null)
+ parameters.setParameter(LuceneConfig.PARAM_MAXIMUMDOCUMENTLENGTH, maximumDocumentLength);
return null;
}
Modified: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneDocument.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneDocument.java?rev=1689479&r1=1689478&r2=1689479&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneDocument.java (original)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/java/org/apache/manifoldcf/agents/output/lucene/LuceneDocument.java Mon Jul 6 19:02:37 2015
@@ -32,13 +32,13 @@ public class LuceneDocument {
private Document doc;
- private static final FieldType TYPE_STORED_WITH_TV = new FieldType(TextField.TYPE_STORED);
- static {
- TYPE_STORED_WITH_TV.setStoreTermVectors(true);
- TYPE_STORED_WITH_TV.setStoreTermVectorOffsets(true);
- TYPE_STORED_WITH_TV.setStoreTermVectorPositions(true);
- TYPE_STORED_WITH_TV.freeze();
- }
+ private static final FieldType TYPE_STORED_WITH_TV = new FieldType(TextField.TYPE_STORED);
+ static {
+ TYPE_STORED_WITH_TV.setStoreTermVectors(true);
+ TYPE_STORED_WITH_TV.setStoreTermVectorOffsets(true);
+ TYPE_STORED_WITH_TV.setStoreTermVectorPositions(true);
+ TYPE_STORED_WITH_TV.freeze();
+ }
private static final FieldType TYPE_NOT_STORED_WITH_TV = new FieldType(TextField.TYPE_NOT_STORED);
static {
Modified: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_en_US.properties?rev=1689479&r1=1689478&r2=1689479&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_en_US.properties (original)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_en_US.properties Mon Jul 6 19:02:37 2015
@@ -25,6 +25,7 @@ LuceneConnector.Analyzers=Analyzers
LuceneConnector.Fields=Fields
LuceneConnector.Idfield=id field name
LuceneConnector.Contentfield=content field name
+LuceneConnector.Maximumdocumentlength=Maximum document length
LuceneConnector.PleaseSupplyValidPath=Path can't be empty. Please supply a valid path
@@ -35,6 +36,6 @@ LuceneConnector.PleaseSupplyValidAnalyze
LuceneConnector.PleaseSupplyValidFields=Fields can't be empty. Please supply a valid fields
LuceneConnector.PleaseSupplyValidIdfield=Idfield can't be empty. Please supply a valid idfield
LuceneConnector.PleaseSupplyValidContentfield=Contentfield can't be empty. Please supply a valid contentfield
-
-
+LuceneConnector.PleaseSupplyValidMaximumdocumentlength=Maximum document length can't be empty. Please supply a valid maximum document length
+LuceneConnector.MaximumDocumentLengthMustBeAnInteger=Maximum document length must be an integer
Modified: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_ja_JP.properties?rev=1689479&r1=1689478&r2=1689479&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_ja_JP.properties (original)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_ja_JP.properties Mon Jul 6 19:02:37 2015
@@ -25,6 +25,7 @@ LuceneConnector.Analyzers=Analyzers
LuceneConnector.Fields=Fields
LuceneConnector.Idfield=id field name
LuceneConnector.Contentfield=content field name
+LuceneConnector.Maximumdocumentlength=Maximum document length
LuceneConnector.PleaseSupplyValidPath=Path can't be empty. Please supply a valid path
@@ -35,6 +36,6 @@ LuceneConnector.PleaseSupplyValidAnalyze
LuceneConnector.PleaseSupplyValidFields=Fields can't be empty. Please supply a valid fields
LuceneConnector.PleaseSupplyValidIdfield=Idfield can't be empty. Please supply a valid idfield
LuceneConnector.PleaseSupplyValidContentfield=Contentfield can't be empty. Please supply a valid contentfield
-
-
+LuceneConnector.PleaseSupplyValidMaximumdocumentlength=Maximum document length can't be empty. Please supply a valid maximum document length
+LuceneConnector.MaximumDocumentLengthMustBeAnInteger=Maximum document length must be an integer
Modified: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_zh_CN.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_zh_CN.properties?rev=1689479&r1=1689478&r2=1689479&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_zh_CN.properties (original)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/lucene/common_zh_CN.properties Mon Jul 6 19:02:37 2015
@@ -25,6 +25,7 @@ LuceneConnector.Analyzers=Analyzers
LuceneConnector.Fields=Fields
LuceneConnector.Idfield=id field name
LuceneConnector.Contentfield=content field name
+LuceneConnector.Maximumdocumentlength=Maximum document length
LuceneConnector.PleaseSupplyValidPath=Path can't be empty. Please supply a valid path
@@ -35,6 +36,6 @@ LuceneConnector.PleaseSupplyValidAnalyze
LuceneConnector.PleaseSupplyValidFields=Fields can't be empty. Please supply a valid fields
LuceneConnector.PleaseSupplyValidIdfield=Idfield can't be empty. Please supply a valid idfield
LuceneConnector.PleaseSupplyValidContentfield=Contentfield can't be empty. Please supply a valid contentfield
-
-
+LuceneConnector.PleaseSupplyValidMaximumdocumentlength=Maximum document length can't be empty. Please supply a valid maximum document length
+LuceneConnector.MaximumDocumentLengthMustBeAnInteger=Maximum document length must be an integer
Modified: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/lucene/editConfiguration.js
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/lucene/editConfiguration.js?rev=1689479&r1=1689478&r2=1689479&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/lucene/editConfiguration.js (original)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/lucene/editConfiguration.js Mon Jul 6 19:02:37 2015
@@ -69,11 +69,23 @@ function checkConfig() {
}
if (editconnection.contentfield) {
if (editconnection.contentfield.value == "") {
- alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('LuceneConnector.PleaseSupplyValidIdfield'))");
+ alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('LuceneConnector.PleaseSupplyValidContentfield'))");
editconnection.contentfield.focus();
return false;
}
}
+ if (editconnection.maximumdocumentlength) {
+ if (editconnection.maximumdocumentlength.value == "") {
+ alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('LuceneConnector.PleaseSupplyValidMaximumdocumentlength'))");
+ editconnection.maximumdocumentlength.focus();
+ return false;
+ }
+ if (editconnection.maximumdocumentlength.value != "" && !isInteger(editconnection.maximumdocumentlength.value)) {
+ alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('LuceneConnector.MaximumDocumentLengthMustBeAnInteger'))");
+ editconnection.maximumdocumentlength.focus();
+ return false;
+ }
+ }
return true;
}
@@ -142,6 +154,20 @@ function checkConfigForSave() {
return false;
}
}
+ if (editconnection.maximumdocumentlength) {
+ if (editconnection.maximumdocumentlength.value == "") {
+ alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('LuceneConnector.PleaseSupplyValidMaximumdocumentlength'))");
+ SelectTab("$Encoder.javascriptBodyEscape($ResourceBundle.getString('LuceneConnector.Parameters'))");
+ editconnection.maximumdocumentlength.focus();
+ return false;
+ }
+ if (editconnection.maximumdocumentlength.value != "" && !isInteger(editconnection.maximumdocumentlength.value)) {
+ alert("$Encoder.bodyJavascriptEscape($ResourceBundle.getString('LuceneConnector.MaximumDocumentLengthMustBeAnInteger'))");
+ SelectTab("$Encoder.javascriptBodyEscape($ResourceBundle.getString('LuceneConnector.Parameters'))");
+ editconnection.maximumdocumentlength.focus();
+ return false;
+ }
+ }
return true;
}
//-->
Modified: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/lucene/editConfiguration_Parameters.html
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/lucene/editConfiguration_Parameters.html?rev=1689479&r1=1689478&r2=1689479&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/lucene/editConfiguration_Parameters.html (original)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/lucene/editConfiguration_Parameters.html Mon Jul 6 19:02:37 2015
@@ -82,6 +82,14 @@
value="$Encoder.attributeEscape($CONTENTFIELD)" size="48" />
</td>
</tr>
+ <tr>
+ <td class="description">
+ $Encoder.bodyEscape($ResourceBundle.getString('LuceneConnector.Maximumdocumentlength'))
+ </td>
+ <td class="value"><input name="maximumdocumentlength" type="text"
+ value="$Encoder.attributeEscape($MAXIMUMDOCUMENTLENGTH)" size="48" />
+ </td>
+ </tr>
</table>
#else
@@ -94,5 +102,6 @@
<input type="hidden" name="fields" value="$Encoder.attributeEscape($FIELDS)" />
<input type="hidden" name="idfield" value="$Encoder.attributeEscape($IDFIELD)" />
<input type="hidden" name="contentfield" value="$Encoder.attributeEscape($CONTENTFIELD)" />
+<input type="hidden" name="maximumdocumentlength" value="$Encoder.attributeEscape($MAXIMUMDOCUMENTLENGTH)" />
#end
Modified: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/lucene/viewConfiguration.html
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/lucene/viewConfiguration.html?rev=1689479&r1=1689478&r2=1689479&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/lucene/viewConfiguration.html (original)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/main/resources/org/apache/manifoldcf/agents/output/lucene/viewConfiguration.html Mon Jul 6 19:02:37 2015
@@ -48,4 +48,8 @@
<td class="description">$Encoder.bodyEscape($ResourceBundle.getString('LuceneConnector.Contentfield'))</td>
<td class="value">$Encoder.bodyEscape($CONTENTFIELD)</td>
</tr>
+ <tr>
+ <td class="description">$Encoder.bodyEscape($ResourceBundle.getString('LuceneConnector.Maximumdocumentlength'))</td>
+ <td class="value">$Encoder.bodyEscape($MAXIMUMDOCUMENTLENGTH)</td>
+ </tr>
</table>
\ No newline at end of file
Modified: manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/test/java/org/apache/manifoldcf/agents/output/lucene/tests/LuceneClientTest.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/test/java/org/apache/manifoldcf/agents/output/lucene/tests/LuceneClientTest.java?rev=1689479&r1=1689478&r2=1689479&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/test/java/org/apache/manifoldcf/agents/output/lucene/tests/LuceneClientTest.java (original)
+++ manifoldcf/branches/CONNECTORS-1219/connectors/lucene/connector/src/test/java/org/apache/manifoldcf/agents/output/lucene/tests/LuceneClientTest.java Mon Jul 6 19:02:37 2015
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.Term;
@@ -116,12 +115,12 @@ public class LuceneClientTest {
LuceneClient client1 =
LuceneClientManager.getClient(path, LuceneClient.defaultCharfilters(), LuceneClient.defaultTokenizers(), LuceneClient.defaultFilters(), LuceneClient.defaultAnalyzers(), LuceneClient.defaultFields(),
- LuceneClient.defaultIdField(), LuceneClient.defaultContentField());
+ LuceneClient.defaultIdField(), LuceneClient.defaultContentField(), LuceneClient.defaultMaximumDocumentLength());
assertThat(client1.isOpen(), is(true));
LuceneClient client2 =
LuceneClientManager.getClient(path, LuceneClient.defaultCharfilters(), LuceneClient.defaultTokenizers(), LuceneClient.defaultFilters(), LuceneClient.defaultAnalyzers(), LuceneClient.defaultFields(),
- "id", "content");
+ "id", "content", LuceneClient.defaultMaximumDocumentLength());
assertThat(client2.isOpen(), is(true));
assertThat(client1, is(client2));
@@ -130,7 +129,7 @@ public class LuceneClientTest {
try {
client3 =
LuceneClientManager.getClient(path, LuceneClient.defaultCharfilters(), LuceneClient.defaultTokenizers(), LuceneClient.defaultFilters(), LuceneClient.defaultAnalyzers(), LuceneClient.defaultFields(),
- "dummy_id", "dummy_content");
+ "dummy_id", "dummy_content", LuceneClient.defaultMaximumDocumentLength());
fail("Should not get here");
} catch (Exception e) {
assert e instanceof IllegalStateException;
@@ -143,7 +142,7 @@ public class LuceneClientTest {
client3 =
LuceneClientManager.getClient(path, LuceneClient.defaultCharfilters(), LuceneClient.defaultTokenizers(), LuceneClient.defaultFilters(), LuceneClient.defaultAnalyzers(), LuceneClient.defaultFields(),
- "dummy_id", "dummy_content");
+ "dummy_id", "dummy_content", LuceneClient.defaultMaximumDocumentLength());
assertThat(client3.isOpen(), is(true));
assertThat(client3, not(client1));