You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2018/11/30 23:34:01 UTC
svn commit: r1847863 - in
/manifoldcf/trunk/connectors/elasticsearch/connector/src/main:
java/org/apache/manifoldcf/agents/output/elasticsearch/
native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/
resources/org/apache/manifoldcf/agents/outp...
Author: kwright
Date: Fri Nov 30 23:34:01 2018
New Revision: 1847863
URL: http://svn.apache.org/viewvc?rev=1847863&view=rev
Log:
Add basic auth security to ElasticSearch connector. Committed on behalf of Steph van Schalkwyk.
Modified:
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_fr_FR.properties
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html
manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java Fri Nov 30 23:34:01 2018
@@ -26,7 +26,7 @@ import org.apache.http.client.HttpClient
import java.io.IOException;
import java.util.Locale;
-import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
+//import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
import org.apache.manifoldcf.crawler.system.Logging;
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java Fri Nov 30 23:34:01 2018
@@ -35,13 +35,16 @@ public class ElasticSearchConfig extends
/** Parameters used for the configuration */
final private static ParameterEnum[] CONFIGURATIONLIST =
{
- ParameterEnum.SERVERVERSION,
+// ParameterEnum.SERVERVERSION,
ParameterEnum.SERVERLOCATION,
+ ParameterEnum.USERNAME,
+ ParameterEnum.PASSWORD,
ParameterEnum.INDEXNAME,
ParameterEnum.INDEXTYPE,
ParameterEnum.USEMAPPERATTACHMENTS,
ParameterEnum.PIPELINENAME,
ParameterEnum.CONTENTATTRIBUTENAME,
+ ParameterEnum.URIATTRIBUTENAME,
ParameterEnum.CREATEDDATEATTRIBUTENAME,
ParameterEnum.MODIFIEDDATEATTRIBUTENAME,
ParameterEnum.INDEXINGDATEATTRIBUTENAME,
@@ -59,9 +62,19 @@ public class ElasticSearchConfig extends
super(CONFIGURATIONLIST);
for (ParameterEnum param : CONFIGURATIONLIST)
{
- String value = params.getParameter(param.name());
+ final boolean isPassword = param.name().endsWith("PASSWORD");
+ String value;
+ if (isPassword) {
+ value = params.getObfuscatedParameter(param.name());
+ }
+ else
+ {
+ value = params.getParameter(param.name());
+ }
if (value == null)
+ {
value = param.defaultValue;
+ }
put(param, value);
}
}
@@ -82,9 +95,20 @@ public class ElasticSearchConfig extends
{
for (ParameterEnum param : CONFIGURATIONLIST)
{
- String p = variableContext.getParameter(param.name().toLowerCase(Locale.ROOT));
+ final String paramName = param.name().toLowerCase(Locale.ROOT);
+ final boolean isPassword = param.name().endsWith("PASSWORD");
+ String p = variableContext.getParameter(paramName);
if (p != null)
- parameters.setParameter(param.name(), p);
+ {
+ if (isPassword)
+ {
+ parameters.setObfuscatedParameter(param.name(), p);
+ }
+ else
+ {
+ parameters.setParameter(param.name(), p);
+ }
+ }
}
String useMapperAttachmentsPresent = variableContext.getParameter("usemapperattachments_present");
@@ -97,40 +121,19 @@ public class ElasticSearchConfig extends
}
}
- final public boolean isServerAfter5()
+ final public String getServerLocation()
{
- return getMajorVersion() >= 5;
- }
-
- final public int getMajorVersion() {
- final String version = getServerVersion();
- if (version == null || version.length() == 0) {
- return 0;
- }
- final int index = version.indexOf(".");
- final String upperVersion;
- if (index == -1) {
- upperVersion = version;
- } else {
- upperVersion = version.substring(0, index);
- }
- int majorVersion;
- try {
- majorVersion = Integer.parseInt(upperVersion);
- } catch (Exception e) {
- majorVersion = 0;
- }
- return majorVersion;
- }
+ return get(ParameterEnum.SERVERLOCATION);
+ }
- final public String getServerVersion()
+ final public String getUserName()
{
- return get(ParameterEnum.SERVERVERSION);
+ return get(ParameterEnum.USERNAME);
}
- final public String getServerLocation()
+ final public String getPassword()
{
- return get(ParameterEnum.SERVERLOCATION);
+ return get(ParameterEnum.PASSWORD);
}
final public String getIndexName()
@@ -158,6 +161,11 @@ public class ElasticSearchConfig extends
return get(ParameterEnum.CONTENTATTRIBUTENAME);
}
+ final public String getUriAttributeName()
+ {
+ return get(ParameterEnum.URIATTRIBUTENAME);
+ }
+
final public String getCreatedDateAttributeName()
{
return get(ParameterEnum.CREATEDDATEATTRIBUTENAME);
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java Fri Nov 30 23:34:01 2018
@@ -40,21 +40,30 @@ import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.Header;
import org.apache.http.ProtocolVersion;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.Credentials;
+import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.util.EntityUtils;
import org.apache.http.message.BasicHeader;
import org.apache.http.entity.ContentType;
-
+import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.impl.client.BasicAuthCache;
+import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.client.RedirectException;
+import org.apache.http.client.AuthCache;
import org.apache.http.client.CircularRedirectException;
+import org.apache.http.client.CredentialsProvider;
import org.apache.http.NoHttpResponseException;
import org.apache.http.HttpException;
+import org.apache.http.HttpHost;
import org.apache.http.ParseException;
import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
import org.apache.manifoldcf.core.util.URLEncoder;
+import org.apache.manifoldcf.crawler.system.Logging;
public class ElasticSearchConnection
{
@@ -66,6 +75,10 @@ public class ElasticSearchConnection
private String indexName;
+ private String userName;
+
+ private String userPassword;
+
private String resultDescription;
private String callUrlSnippet;
@@ -93,18 +106,38 @@ public class ElasticSearchConnection
callUrlSnippet = null;
serverLocation = config.getServerLocation();
indexName = config.getIndexName();
+ userName = config.getUserName();
+ userPassword = config.getPassword();
}
-
protected StringBuffer getApiUrl(String command, boolean checkConnection) throws ManifoldCFException
{
- StringBuffer url = new StringBuffer(serverLocation);
+ String basicAuth = "";
+ if (userName.length() > 0 && userPassword.length() >0) {
+ basicAuth = userName + ":" + userPassword + "@";
+ }
+ Logging.connectors.debug("Auth: " + basicAuth);
+
+ String[] serverLocationSplit = serverLocation.split("://",2);
+ StringBuffer url;
+ if (basicAuth.length() > 0) {
+ url = new StringBuffer(serverLocationSplit[0] + "://" + basicAuth + serverLocationSplit[1]);
+ }
+ else {
+ url = new StringBuffer(serverLocationSplit[0] + "://" + basicAuth + serverLocationSplit[1]);
+ }
if (!serverLocation.endsWith("/"))
url.append('/');
+
+ Logging.connectors.debug("Url: " + url);
+
if(!checkConnection)
url.append(URLEncoder.encode(indexName)).append("/");
url.append(command);
callUrlSnippet = url.toString();
+
+ Logging.connectors.debug("UrlEnc: " + url);
+
return url;
}
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java Fri Nov 30 23:34:01 2018
@@ -19,7 +19,7 @@
package org.apache.manifoldcf.agents.output.elasticsearch;
-import java.io.File;
+//import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
@@ -27,26 +27,26 @@ import java.util.Locale;
import java.util.Map;
import java.util.Iterator;
import java.util.HashMap;
-import java.util.concurrent.TimeUnit;
+//import java.util.concurrent.TimeUnit;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.client.HttpClient;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
-import org.apache.http.impl.client.HttpClientBuilder;
+//import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.protocol.HttpRequestExecutor;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.config.SocketConfig;
-import org.apache.http.client.HttpRequestRetryHandler;
-import org.apache.http.protocol.HttpContext;
+//import org.apache.http.client.HttpRequestRetryHandler;
+//import org.apache.http.protocol.HttpContext;
-import org.apache.commons.io.FilenameUtils;
+//import org.apache.commons.io.FilenameUtils;
import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
-import org.apache.manifoldcf.agents.interfaces.IOutputNotifyActivity;
+//import org.apache.manifoldcf.agents.interfaces.IOutputNotifyActivity;
import org.apache.manifoldcf.agents.interfaces.IOutputRemoveActivity;
-import org.apache.manifoldcf.agents.interfaces.IOutputCheckActivity;
+//import org.apache.manifoldcf.agents.interfaces.IOutputCheckActivity;
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
import org.apache.manifoldcf.agents.output.BaseOutputConnector;
@@ -54,12 +54,12 @@ import org.apache.manifoldcf.agents.outp
import org.apache.manifoldcf.agents.output.elasticsearch.ElasticSearchConnection.Result;
import org.apache.manifoldcf.core.interfaces.Specification;
import org.apache.manifoldcf.core.interfaces.ConfigParams;
-import org.apache.manifoldcf.core.interfaces.ConfigurationNode;
+//import org.apache.manifoldcf.core.interfaces.ConfigurationNode;
import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
import org.apache.manifoldcf.core.interfaces.IPostParameters;
import org.apache.manifoldcf.core.interfaces.IThreadContext;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
-import org.apache.manifoldcf.core.interfaces.SpecificationNode;
+//import org.apache.manifoldcf.core.interfaces.SpecificationNode;
import org.apache.manifoldcf.core.interfaces.VersionContext;
/**
@@ -72,11 +72,9 @@ public class ElasticSearchConnector exte
private final static String ELASTICSEARCH_INDEXATION_ACTIVITY = "Indexation";
private final static String ELASTICSEARCH_DELETION_ACTIVITY = "Deletion";
- private final static String ELASTICSEARCH_OPTIMIZE_ACTIVITY = "Optimize";
private final static String[] ELASTICSEARCH_ACTIVITIES =
- { ELASTICSEARCH_INDEXATION_ACTIVITY, ELASTICSEARCH_DELETION_ACTIVITY,
- ELASTICSEARCH_OPTIMIZE_ACTIVITY };
+ { ELASTICSEARCH_INDEXATION_ACTIVITY, ELASTICSEARCH_DELETION_ACTIVITY };
private final static String ELASTICSEARCH_TAB_PARAMETERS = "ElasticSearchConnector.Parameters";
@@ -210,7 +208,7 @@ public class ElasticSearchConnector exte
{
Map<String,String> paramMap = null;
if (params != null) {
- paramMap = params.buildMap();
+ paramMap = params.buildMap(out);
if (tabName != null) {
paramMap.put("TabName", tabName);
}
@@ -423,28 +421,4 @@ public class ElasticSearchConnector exte
}
}
- @Override
- public void noteJobComplete(IOutputNotifyActivity activities)
- throws ManifoldCFException, ServiceInterruption
- {
- ElasticSearchConfig config = getConfigParameters(null);
- HttpClient client = getSession();
- long startTime = System.currentTimeMillis();
- ElasticSearchAction oo = new ElasticSearchAction(client, config);
- try
- {
- if (config.isServerAfter5()) {
- oo.executePOST(CommandEnum._forcemerge, false);
- } else {
- oo.executeGET(CommandEnum._optimize, false);
- }
- }
- finally
- {
- activities.recordActivity(startTime, ELASTICSEARCH_OPTIMIZE_ACTIVITY, null,
- oo.getCallUrlSnippet(), oo.getResultCode(),
- oo.getResultDescription());
- }
- }
-
}
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java Fri Nov 30 23:34:01 2018
@@ -19,15 +19,18 @@
package org.apache.manifoldcf.agents.output.elasticsearch;
+import org.apache.http.Header;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpDelete;
-
-import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
+import org.apache.manifoldcf.core.common.Base64;
+//import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
import org.apache.manifoldcf.core.util.URLEncoder;
import org.apache.manifoldcf.crawler.system.Logging;
+//import java.util.Locale;
+
public class ElasticSearchDelete extends ElasticSearchConnection
{
@@ -43,6 +46,16 @@ public class ElasticSearchDelete extends
HttpDelete method = new HttpDelete(config.getServerLocation() +
"/" + config.getIndexName() + "/" + config.getIndexType()
+ "/" + idField);
+
+ if (config.getUserName().length() > 0 && config.getPassword().length() >0) {
+ byte[] basicAuth = (config.getUserName() + ":" + config.getPassword()).getBytes();
+ Base64 encoder = new Base64();
+ String encoding = encoder.encodeByteArray(basicAuth);
+ method.setHeader("Authorization", "Basic " + encoding);
+ Header header = method.getLastHeader("Authorization");
+ Logging.connectors.debug("Header: " + new String(header.getValue().getBytes()));
+ }
+
call(method);
String error = checkJson(jsonException);
if (getResult() == Result.OK && error == null)
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java Fri Nov 30 23:34:01 2018
@@ -32,6 +32,7 @@ import java.util.Iterator;
import java.text.SimpleDateFormat;
import java.util.Locale;
import java.util.TimeZone;
+
import java.util.Date;
import org.apache.http.client.HttpClient;
@@ -42,10 +43,10 @@ import org.apache.http.util.EntityUtils;
import org.apache.http.message.BasicHeader;
import org.apache.http.Header;
import org.apache.commons.io.IOUtils;
-import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
+//import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
-import org.apache.manifoldcf.agents.output.elasticsearch.ElasticSearchConnection.Result;
+//import org.apache.manifoldcf.agents.output.elasticsearch.ElasticSearchConnection.Result;
import org.apache.manifoldcf.core.common.Base64;
import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
import org.apache.manifoldcf.core.util.URLEncoder;
@@ -159,7 +160,6 @@ public class ElasticSearchIndex extends
needComma = writeField(pw, needComma, fieldName, fieldValues);
}
}
-
// Standard document fields
final Date createdDate = document.getCreatedDate();
if (createdDate != null && createdDateAttributeName != null && createdDateAttributeName.length() > 0)
@@ -181,7 +181,6 @@ public class ElasticSearchIndex extends
{
needComma = writeField(pw, needComma, mimeTypeAttributeName, new String[]{mimeType});
}
-
needComma = writeACLs(pw, needComma, "document", acls, denyAcls);
needComma = writeACLs(pw, needComma, "share", shareAcls, shareDenyAcls);
needComma = writeACLs(pw, needComma, "parent", parentAcls, parentDenyAcls);
@@ -245,7 +244,6 @@ public class ElasticSearchIndex extends
}
}
}
-
pw.append("\"");
needComma = true;
}
@@ -420,16 +418,13 @@ public class ElasticSearchIndex extends
String[] acls, String[] denyAcls, String[] shareAcls, String[] shareDenyAcls, String[] parentAcls, String[] parentDenyAcls)
throws ManifoldCFException, ServiceInterruption
{
-
-
final String idField = URLEncoder.encode(documentURI);
final String encodedPipelineName = (config.getPipelineName() == null || config.getPipelineName().length() == 0)?null:URLEncoder.encode(config.getPipelineName());
-
- final String command = config.getIndexType() + "/" + idField;
+ final String command = config.getIndexType() + "/" + idField;
final String fullCommand = (encodedPipelineName == null)?command:(command + "?pipeline=" + encodedPipelineName);
-
StringBuffer url = getApiUrl(fullCommand, false);
HttpPut put = new HttpPut(url.toString());
+ Logging.connectors.debug("HttPutUri: " + url.toString());
put.setEntity(new IndexRequestEntity(document, inputStream,
acls, denyAcls, shareAcls, shareDenyAcls, parentAcls, parentDenyAcls,
config.getUseMapperAttachments(),
@@ -438,13 +433,23 @@ public class ElasticSearchIndex extends
config.getModifiedDateAttributeName(),
config.getIndexingDateAttributeName(),
config.getMimeTypeAttributeName()));
+
+ if (config.getUserName().length() > 0 && config.getPassword().length() >0) {
+ byte[] basicAuth = (config.getUserName() + ":" + config.getPassword()).getBytes();
+ Base64 encoder = new Base64();
+ String encoding = encoder.encodeByteArray(basicAuth);
+ put.setHeader("Authorization", "Basic " + encoding);
+ Header header = put.getLastHeader("Authorization");
+ Logging.connectors.debug("Header: " + new String(header.getValue().getBytes()));
+ }
+
if (call(put) == false)
return false;
String error = checkJson(jsonException);
if (getResult() == Result.OK && error == null)
return true;
setResult("JSONERROR",Result.ERROR, error);
- Logging.connectors.warn("ES: Index failed: "+getResponse());
+ Logging.connectors.warn("ES: Index failed: " + getResponse());
return true;
}
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java Fri Nov 30 23:34:01 2018
@@ -22,6 +22,8 @@ package org.apache.manifoldcf.agents.out
import java.util.HashMap;
import java.util.Map;
+import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
+
import org.apache.manifoldcf.agents.output.elasticsearch.ElasticSearchParam.ParameterEnum;
/**
@@ -33,27 +35,31 @@ public class ElasticSearchParam extends
/** Parameters constants */
public enum ParameterEnum
{
- SERVERVERSION(""),
-
SERVERLOCATION("http://localhost:9200/"),
INDEXNAME("index"),
- INDEXTYPE("generictype"),
+ USERNAME(""),
+
+ PASSWORD(""),
+
+ INDEXTYPE("generic"),
- USEMAPPERATTACHMENTS("true"),
+ USEMAPPERATTACHMENTS("false"),
PIPELINENAME(""),
- CONTENTATTRIBUTENAME(""),
+ CONTENTATTRIBUTENAME("content"),
- CREATEDDATEATTRIBUTENAME(""),
+ URIATTRIBUTENAME("url"),
+
+ CREATEDDATEATTRIBUTENAME("created"),
- MODIFIEDDATEATTRIBUTENAME(""),
+ MODIFIEDDATEATTRIBUTENAME("last-modified"),
- INDEXINGDATEATTRIBUTENAME(""),
+ INDEXINGDATEATTRIBUTENAME("indexed"),
- MIMETYPEATTRIBUTENAME(""),
+ MIMETYPEATTRIBUTENAME("mime-type"),
FIELDLIST("");
@@ -72,11 +78,23 @@ public class ElasticSearchParam extends
super(params.length);
}
- final public Map<String, String> buildMap()
+ final public Map<String, String> buildMap(IHTTPOutput out)
{
Map<String, String> rval = new HashMap<String, String>();
for (Map.Entry<ParameterEnum, String> entry : this.entrySet())
- rval.put(entry.getKey().name(), entry.getValue());
+ {
+ final String key = entry.getKey().name();
+ final boolean isPassword = key.endsWith("PASSWORD");
+ if (isPassword)
+ {
+ // Do not put passwords in plain text in forms
+ rval.put(key, out.mapPasswordToKey(entry.getValue()));
+ }
+ else
+ {
+ rval.put(key, entry.getValue());
+ }
+ }
return rval;
}
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties Fri Nov 30 23:34:01 2018
@@ -15,9 +15,11 @@
ElasticSearchConnector.Parameters=Parameters
-ElasticSearchConnector.ServerVersion=Server version:
+#ElasticSearchConnector.ServerVersion=Server version:
ElasticSearchConnector.ServerLocation=Server location
ElasticSearchConnector.URLColon=(URL):
+ElasticSearchConnector.UserNameColon=User name:
+ElasticSearchConnector.PasswordColon=User password:
ElasticSearchConnector.IndexNameColon=Index name:
ElasticSearchConnector.IndexTypeColon=Index type:
ElasticSearchConnector.UseMapperAttachmentsColon=Use mapper-attachments:
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties Fri Nov 30 23:34:01 2018
@@ -15,9 +15,11 @@
ElasticSearchConnector.Parameters=Parámetros
-ElasticSearchConnector.ServerVersion=Server version:
+#ElasticSearchConnector.ServerVersion=Server version:
ElasticSearchConnector.ServerLocation=Ubicación del servidor
ElasticSearchConnector.URLColon=(URL):
+ElasticSearchConnector.UserNameColon=User name:
+ElasticSearchConnector.PasswordColon=User password:
ElasticSearchConnector.IndexNameColon=El nombre de Ãndice:
ElasticSearchConnector.IndexTypeColon=tipo de Ãndice:
ElasticSearchConnector.UseMapperAttachmentsColon=Use mapper-attachments:
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_fr_FR.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_fr_FR.properties?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_fr_FR.properties (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_fr_FR.properties Fri Nov 30 23:34:01 2018
@@ -17,6 +17,8 @@ ElasticSearchConnector.Parameters=Param�
ElasticSearchConnector.ServerLocation=Emplacement du serveur
ElasticSearchConnector.URLColon=(URL):
+ElasticSearchConnector.UserNameColon=User name:
+ElasticSearchConnector.PasswordColon=User password:
ElasticSearchConnector.IndexNameColon=Nom de l'index:
ElasticSearchConnector.IndexTypeColon=Type d'index:
@@ -28,4 +30,6 @@ ElasticSearchConnector.ElasticSearch=Ela
ElasticSearchConnector.MaxFileSizeBytesColon=Taille max des fichiers (octets):
ElasticSearchConnector.AllowedMIMETypesColon=Types MIME admis:
-ElasticSearchConnector.AllowedFileExtensionsColon=Extensions de fichiers admises\:
\ No newline at end of file
+ElasticSearchConnector.AllowedFileExtensionsColon=Extensions de fichiers admises\:
+
+ElasticSearchConnector.UriAttributeNameColon=Uri field name:
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties Fri Nov 30 23:34:01 2018
@@ -15,9 +15,11 @@
ElasticSearchConnector.Parameters=å¼æ°
-ElasticSearchConnector.ServerVersion=Server version:
+#ElasticSearchConnector.ServerVersion=Server version:
ElasticSearchConnector.ServerLocation=ãµã¼ã
ElasticSearchConnector.URLColon=(URL):
+ElasticSearchConnector.UserNameColon=User name:
+ElasticSearchConnector.PasswordColon=User password:
ElasticSearchConnector.IndexNameColon=ã¤ã³ããã¯ã¹åï¼
ElasticSearchConnector.IndexTypeColon=ã¿ã¤ãåï¼
ElasticSearchConnector.UseMapperAttachmentsColon=Use mapper-attachments:
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties Fri Nov 30 23:34:01 2018
@@ -15,9 +15,11 @@
ElasticSearchConnector.Parameters=åæ°
-ElasticSearchConnector.ServerVersion=Server version:
+#ElasticSearchConnector.ServerVersion=Server version:
ElasticSearchConnector.ServerLocation=æå¡å¨
ElasticSearchConnector.URLColon=(URL):
+ElasticSearchConnector.UserNameColon=User name:
+ElasticSearchConnector.PasswordColon=User password:
ElasticSearchConnector.IndexNameColon=ç´¢å¼å:
ElasticSearchConnector.IndexTypeColon=ç´¢å¼ç±»å:
ElasticSearchConnector.UseMapperAttachmentsColon=Use mapper-attachments:
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html Fri Nov 30 23:34:01 2018
@@ -20,32 +20,33 @@
<table class="displaytable">
<tr>
<td class="description">
- <nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.ServerVersion'))</nobr>
+ <nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.ServerLocation'))</nobr>
+ $Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.URLColon'))
</td>
- <td class="value"><input name="serverversion" type="text"
- value="$Encoder.attributeEscape($SERVERVERSION)" size="10" />
+ <td class="value"><input name="serverlocation" type="text"
+ value="$Encoder.attributeEscape($SERVERLOCATION)" size="48" />
</td>
</tr>
<tr>
<td class="description">
- <nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.ServerLocation'))</nobr>
- $Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.URLColon'))
+ <nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.UserNameColon'))</nobr></td>
+ <td class="value"><input name="username" type="text" value="$Encoder.attributeEscape($USERNAME)" size="24" />
</td>
- <td class="value"><input name="serverlocation" type="text"
- value="$Encoder.attributeEscape($SERVERLOCATION)" size="48" />
+ </tr>
+ <tr>
+ <td class="description">
+ <nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.PasswordColon'))</nobr></td>
+ <td class="value"><input name="password" type="password" value="$Encoder.attributeEscape($PASSWORD)" size="24" />
</td>
</tr>
<tr>
<td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.IndexNameColon'))</nobr></td>
- <td class="value"><input name="indexname" type="text" value="$Encoder.attributeEscape($INDEXNAME)"
- size="24" /></td>
+ <td class="value"><input name="indexname" type="text" value="$Encoder.attributeEscape($INDEXNAME)" size="24" /></td>
</tr>
<tr>
<td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.IndexTypeColon'))</nobr></td>
- <td class="value"><input name="indextype" type="text" value="$Encoder.attributeEscape($INDEXTYPE)"
- size="24" /></td>
+ <td class="value"><input name="indextype" type="text" value="$Encoder.attributeEscape($INDEXTYPE)" size="24" /></td>
</tr>
-
<tr>
<td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.UseMapperAttachmentsColon'))</nobr></td>
<td class="value">
@@ -63,7 +64,6 @@
<td class="value"><input name="pipelinename" type="text" value="$Encoder.attributeEscape($PIPELINENAME)"
size="24" /></td>
</tr>
-
<tr>
<td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.ContentAttributeNameColon'))</nobr></td>
<td class="value"><input name="contentattributename" type="text" value="$Encoder.attributeEscape($CONTENTATTRIBUTENAME)"
@@ -93,8 +93,6 @@
</table>
#else
-
-<input type="hidden" name="serverversion" value="$Encoder.attributeEscape($SERVERVERSION)" />
<input type="hidden" name="serverlocation" value="$Encoder.attributeEscape($SERVERLOCATION)" />
<input type="hidden" name="indexname" value="$Encoder.attributeEscape($INDEXNAME)" />
<input type="hidden" name="indextype" value="$Encoder.attributeEscape($INDEXTYPE)" />
@@ -103,6 +101,7 @@
<input type="hidden" name="usemapperattachments" value="$Encoder.bodyEscape($USEMAPPERATTACHMENTS)" />
<input type="hidden" name="pipelinename" value="$Encoder.attributeEscape($PIPELINENAME)" />
<input type="hidden" name="contentattributename" value="$Encoder.attributeEscape($CONTENTATTRIBUTENAME)" />
+<input type="hidden" name="uriattributename" value="$Encoder.attributeEscape($URIATTRIBUTENAME)" />
<input type="hidden" name="createddateattributename" value="$Encoder.attributeEscape($CREATEDDATEATTRIBUTENAME)" />
<input type="hidden" name="modifieddateattributename" value="$Encoder.attributeEscape($MODIFIEDDATEATTRIBUTENAME)" />
<input type="hidden" name="indexingdateattributename" value="$Encoder.attributeEscape($INDEXINGDATEATTRIBUTENAME)" />
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html Fri Nov 30 23:34:01 2018
@@ -17,15 +17,19 @@
<table class="displaytable">
<tr>
- <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.ServerVersion'))</nobr>
- <td class="value">$Encoder.bodyEscape($SERVERVERSION)</td>
- </tr>
- <tr>
<td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.ServerLocation'))</nobr>
$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.URLColon'))</td>
<td class="value">$Encoder.bodyEscape($SERVERLOCATION)</td>
</tr>
- <tr>
+ <tr>
+ <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.UserNameColon'))</nobr></td>
+ <td class="value">$Encoder.bodyEscape($USERNAME)</td>
+ </tr>
+ <tr>
+ <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.PasswordColon'))</nobr></td>
+ <td class="value">*******</td>
+ </tr>
+ <tr>
<td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.IndexNameColon'))</nobr></td>
<td class="value">$Encoder.bodyEscape($INDEXNAME)</td>
</tr>
@@ -61,5 +65,4 @@
<td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.MimeTypeAttributeNameColon'))</nobr></td>
<td class="value">$Encoder.bodyEscape($MIMETYPEATTRIBUTENAME)</td>
</tr>
-
</table>
\ No newline at end of file