You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2017/06/20 19:25:40 UTC
ambari git commit: AMBARI-21277. ADDENDUM: Fail to create solr
clients in Log Search / Log Feeder (oleewere)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 e239c8cf2 -> a381a9fc4
AMBARI-21277. ADDENDUM: Fail to create solr clients in Log Search / Log Feeder (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a381a9fc
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a381a9fc
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a381a9fc
Branch: refs/heads/branch-2.5
Commit: a381a9fc40f90ac8bdaebe95c95b36f902ba4a00
Parents: e239c8c
Author: oleewere <ol...@gmail.com>
Authored: Tue Jun 20 21:21:05 2017 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Tue Jun 20 21:24:56 2017 +0200
----------------------------------------------------------------------
.../ambari/logfeeder/output/OutputSolr.java | 8 ++--
.../ambari-logsearch-portal/pom.xml | 28 ++++++++++++
.../configurer/SolrCollectionConfigurer.java | 7 ++-
.../logsearch/dao/SolrSchemaFieldDao.java | 45 +++++++++++++-------
.../handler/CreateCollectionHandler.java | 4 +-
5 files changed, 71 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a381a9fc/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
index d37a3bb..7103905 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
@@ -37,6 +37,7 @@ import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.util.DateUtil;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.commons.lang3.StringUtils;
+import org.apache.http.impl.client.SystemDefaultHttpClient;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.solr.client.solrj.SolrClient;
@@ -91,7 +92,6 @@ public class OutputSolr extends Output {
public void init() throws Exception {
super.init();
initParams();
- setupSecurity();
createOutgoingBuffer();
createSolrWorkers();
}
@@ -179,8 +179,10 @@ public class OutputSolr extends Output {
throw new Exception("For solr cloud property collection is mandatory");
}
LOG.info("Using collection=" + collection);
-
- CloudSolrClient solrClient = new CloudSolrClient(zkConnectString);
+ setupSecurity();
+ SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient();
+ HttpClientUtil.configureClient(httpClient, null);
+ CloudSolrClient solrClient = new CloudSolrClient(zkConnectString, httpClient);
solrClient.setDefaultCollection(collection);
return solrClient;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a381a9fc/ambari-logsearch/ambari-logsearch-portal/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/pom.xml b/ambari-logsearch/ambari-logsearch-portal/pom.xml
index ec0bd43..46b4350 100755
--- a/ambari-logsearch/ambari-logsearch-portal/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-portal/pom.xml
@@ -38,6 +38,7 @@
<swagger.version>1.5.8</swagger.version>
<spring-data-solr.version>2.0.2.RELEASE</spring-data-solr.version>
<jjwt.version>0.6.0</jjwt.version>
+ <http-components.version>4.4.1</http-components.version>
</properties>
<profiles>
<!-- Dev Profile Start -->
@@ -523,6 +524,27 @@
<version>${spring.security.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${http-components.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>${http-components.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpmime</artifactId>
+ <version>${http-components.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.glassfish.jersey.ext</groupId>
<artifactId>jersey-spring3</artifactId>
<version>2.23.2</version>
@@ -537,6 +559,12 @@
<groupId>org.glassfish.jersey.connectors</groupId>
<artifactId>jersey-apache-connector</artifactId>
<version>${jersey.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.core</groupId>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a381a9fc/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
index 7259bf8..906cfed 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java
@@ -28,6 +28,9 @@ import org.apache.ambari.logsearch.handler.ReloadCollectionHandler;
import org.apache.ambari.logsearch.handler.UpgradeSchemaHandler;
import org.apache.ambari.logsearch.handler.UploadConfigurationHandler;
import org.apache.commons.lang.StringUtils;
+import org.apache.http.client.HttpClient;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.impl.client.SystemDefaultHttpClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
@@ -123,7 +126,9 @@ public class SolrCollectionConfigurer implements SolrConfigurer {
private CloudSolrClient createClient(String solrUrl, String zookeeperConnectString, String defaultCollection) {
if (StringUtils.isNotEmpty(zookeeperConnectString)) {
- CloudSolrClient cloudSolrClient = new CloudSolrClient(zookeeperConnectString);
+ DefaultHttpClient httpClient = new SystemDefaultHttpClient();
+ HttpClientUtil.configureClient(httpClient, null);
+ CloudSolrClient cloudSolrClient = new CloudSolrClient(zookeeperConnectString, httpClient);
cloudSolrClient.setDefaultCollection(defaultCollection);
return cloudSolrClient;
} else if (StringUtils.isNotEmpty(solrUrl)) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/a381a9fc/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
index 01378c1..26f90e3 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
@@ -22,9 +22,11 @@ import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.common.MessageEnums;
import org.apache.ambari.logsearch.conf.SolrUserPropsConfig;
+import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.SystemDefaultHttpClient;
+import org.apache.http.util.EntityUtils;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
@@ -66,18 +68,18 @@ public class SolrSchemaFieldDao {
@Inject
private AuditSolrDao auditSolrDao;
-
+
@Inject
private SolrUserPropsConfig solrUserConfigPropsConfig;
-
+
private int retryCount;
private int skipCount;
-
+
private Map<String, String> serviceSchemaFieldNameMap = new HashMap<>();
private Map<String, String> serviceSchemaFieldTypeMap = new HashMap<>();
private Map<String, String> auditSchemaFieldNameMap = new HashMap<>();
private Map<String, String> auditSchemaFieldTypeMap = new HashMap<>();
-
+
@Scheduled(fixedDelay = RETRY_SECOND * 1000)
public void populateAllSchemaFields() {
if (skipCount > 0) {
@@ -107,7 +109,7 @@ public class SolrSchemaFieldDao {
schemaRequest.setMethod(SolrRequest.METHOD.GET);
schemaRequest.setPath("/schema");
schemaResponse = schemaRequest.process(solrClient);
-
+
LOG.debug("populateSchemaFields() collection=" + solrClient.getDefaultCollection() + ", luke=" + lukeResponses +
", schema= " + schemaResponse);
} catch (SolrException | SolrServerException | IOException e) {
@@ -127,26 +129,37 @@ public class SolrSchemaFieldDao {
}
}
}
-
+
private static final String LUKE_REQUEST_URL_SUFFIX = "admin/luke?numTerms=0&wt=javabin&version=2";
-
+
@SuppressWarnings("unchecked")
private List<LukeResponse> getLukeResponsesForCores(CloudSolrClient solrClient) {
ZkStateReader zkStateReader = solrClient.getZkStateReader();
Collection<Slice> activeSlices = zkStateReader.getClusterState().getActiveSlices(solrClient.getDefaultCollection());
-
+
List<LukeResponse> lukeResponses = new ArrayList<>();
for (Slice slice : activeSlices) {
for (Replica replica : slice.getReplicas()) {
- try (CloseableHttpClient httpClient = HttpClientUtil.createClient(null)) {
+ HttpEntity httpEntity = null;
+ try(SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient()) {
+ HttpClientUtil.configureClient(httpClient, null);
HttpGet request = new HttpGet(replica.getCoreUrl() + LUKE_REQUEST_URL_SUFFIX);
HttpResponse response = httpClient.execute(request);
- NamedList<Object> lukeData = (NamedList<Object>) new JavaBinCodec(null, null).unmarshal(response.getEntity().getContent());
+ httpEntity = response.getEntity();
+ NamedList<Object> lukeData = (NamedList<Object>) new JavaBinCodec(null, null).unmarshal(httpEntity.getContent());
LukeResponse lukeResponse = new LukeResponse();
lukeResponse.setResponse(lukeData);
lukeResponses.add(lukeResponse);
} catch (IOException e) {
LOG.error("Exception during getting luke responses", e);
+ } finally {
+ if (httpEntity != null) {
+ try {
+ EntityUtils.consume(httpEntity);
+ } catch (IOException e) {
+ LOG.error("Error during consuming http entity...", e);
+ }
+ }
}
}
}
@@ -158,7 +171,7 @@ public class SolrSchemaFieldDao {
try {
HashMap<String, String> _schemaFieldNameMap = new HashMap<>();
HashMap<String, String> _schemaFieldTypeMap = new HashMap<>();
-
+
for (LukeResponse lukeResponse : lukeResponses) {
for (Entry<String, FieldInfo> e : lukeResponse.getFieldInfo().entrySet()) {
String name = e.getKey();
@@ -169,7 +182,7 @@ public class SolrSchemaFieldDao {
}
}
}
-
+
List<FieldTypeDefinition> fieldTypes = schemaResponse.getSchemaRepresentation().getFieldTypes();
for (FieldTypeDefinition fieldType : fieldTypes) {
Map<String, Object> fieldAttributes = fieldType.getAttributes();
@@ -177,7 +190,7 @@ public class SolrSchemaFieldDao {
String fieldTypeJson = new JSONObject(fieldAttributes).toString();
_schemaFieldTypeMap.put(name, fieldTypeJson);
}
-
+
List<Map<String, Object>> fields = schemaResponse.getSchemaRepresentation().getFields();
for (Map<String, Object> field : fields) {
String name = (String) field.get("name");
@@ -187,11 +200,11 @@ public class SolrSchemaFieldDao {
_schemaFieldNameMap.put(name, type);
}
}
-
+
if (_schemaFieldNameMap.isEmpty() || _schemaFieldTypeMap.isEmpty()) {
return;
}
-
+
synchronized (this) {
schemaFieldNameMap.clear();
schemaFieldNameMap.putAll(_schemaFieldNameMap);
http://git-wip-us.apache.org/repos/asf/ambari/blob/a381a9fc/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
index 752a1e1..b7743b1 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
@@ -23,6 +23,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.SystemDefaultHttpClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
@@ -170,7 +171,8 @@ public class CreateCollectionHandler implements SolrZkRequestHandler<Boolean> {
private void updateMaximumNumberOfShardsPerCore(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) throws IOException {
String baseUrl = getRandomBaseUrl(slices);
if (baseUrl != null) {
- CloseableHttpClient httpClient = HttpClientUtil.createClient(null);
+ SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient();
+ HttpClientUtil.configureClient(httpClient, null);
HttpGet request = new HttpGet(baseUrl + String.format(MODIFY_COLLECTION_QUERY,
solrPropsConfig.getCollection(), MAX_SHARDS_PER_NODE, calculateMaxShardsPerNode(solrPropsConfig)));
HttpResponse response = httpClient.execute(request);