You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2017/01/19 13:13:28 UTC
[2/3] ambari git commit: AMBARI-19468. Run ZK operations in logsearch
internally and enable ACL security (oleewere)
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
deleted file mode 100644
index da76924..0000000
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logsearch.dao;
-
-import org.apache.ambari.logsearch.conf.SolrPropsConfig;
-import org.apache.commons.lang.StringUtils;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpClientUtil;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.response.CollectionAdminResponse;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.ROUTER_FIELD;
-
-import javax.inject.Named;
-import javax.ws.rs.core.Response;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-
-@Named
-class SolrCollectionDao {
-
- private static final Logger LOG = LoggerFactory.getLogger(SolrCollectionDao.class);
-
- private static final int SETUP_RETRY_SECOND = 30;
- private static final String MODIFY_COLLECTION_QUERY = "/admin/collections?action=MODIFYCOLLECTION&collection=%s&%s=%d";
- private static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode";
-
- /**
- * This will try to get the collections from the Solr. Ping doesn't work if
- * collection is not given
- */
- boolean checkSolrStatus(CloudSolrClient cloudSolrClient) {
- int waitDurationMS = 3 * 60 * 1000;
- boolean status = false;
- try {
- long beginTimeMS = System.currentTimeMillis();
- long waitIntervalMS = 2000;
- int pingCount = 0;
- while (true) {
- pingCount++;
- try {
- List<String> collectionList = getCollections(cloudSolrClient);
- if (collectionList != null) {
- LOG.info("checkSolrStatus(): Solr getCollections() is success. collectionList=" + collectionList);
- status = true;
- break;
- }
- } catch (Exception ex) {
- LOG.error("Error while doing Solr check", ex);
- }
- if (System.currentTimeMillis() - beginTimeMS > waitDurationMS) {
- LOG.error("Solr is not reachable even after " + (System.currentTimeMillis() - beginTimeMS) + " ms. " +
- "If you are using alias, then you might have to restart LogSearch after Solr is up and running.");
- break;
- } else {
- LOG.warn("Solr is not not reachable yet. getCollections() attempt count=" + pingCount + ". " +
- "Will sleep for " + waitIntervalMS + " ms and try again.");
- }
- Thread.sleep(waitIntervalMS);
-
- }
- } catch (Throwable t) {
- LOG.error("Seems Solr is not up.");
- }
- return status;
- }
-
- void setupCollections(final CloudSolrClient solrClient, final SolrPropsConfig solrPropsConfig) throws Exception {
- boolean setupStatus = createCollectionsIfNeeded(solrClient, solrPropsConfig);
- LOG.info("Setup status for " + solrPropsConfig.getCollection() + " is " + setupStatus);
- if (!setupStatus) {
- // Start a background thread to do setup
- Thread setupThread = new Thread("setup_collection_" + solrPropsConfig.getCollection()) {
- @Override
- public void run() {
- LOG.info("Started monitoring thread to check availability of Solr server. collection=" + solrPropsConfig.getCollection());
- int retryCount = 0;
- while (true) {
- try {
- Thread.sleep(SETUP_RETRY_SECOND * 1000);
- retryCount++;
- boolean setupStatus = createCollectionsIfNeeded(solrClient, solrPropsConfig);
- if (setupStatus) {
- LOG.info("Setup for collection " + solrPropsConfig.getCollection() + " is successful. Exiting setup retry thread");
- break;
- }
- } catch (InterruptedException sleepInterrupted) {
- LOG.info("Sleep interrupted while setting up collection " + solrPropsConfig.getCollection());
- break;
- } catch (Exception e) {
- LOG.error("Error setting up collection=" + solrPropsConfig.getCollection(), e);
- }
- LOG.error("Error setting collection. collection=" + solrPropsConfig.getCollection() + ", retryCount=" + retryCount);
- }
- }
- };
- setupThread.setDaemon(true);
- setupThread.start();
- }
- }
-
- private boolean createCollectionsIfNeeded(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) {
- boolean result = false;
- try {
- List<String> allCollectionList = getCollections(solrClient);
- if (solrPropsConfig.getSplitInterval().equalsIgnoreCase("none")) {
- result = createCollection(solrClient, solrPropsConfig, allCollectionList);
- } else {
- result = setupCollectionsWithImplicitRouting(solrClient, solrPropsConfig, allCollectionList);
- }
- } catch (Exception ex) {
- LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection(), ex);
- }
- return result;
- }
-
- @SuppressWarnings("unchecked")
- List<String> getCollections(CloudSolrClient solrClient) throws SolrServerException,
- IOException {
- try {
- CollectionAdminRequest.List colListReq = new CollectionAdminRequest.List();
- CollectionAdminResponse response = colListReq.process(solrClient);
- if (response.getStatus() != 0) {
- LOG.error("Error getting collection list from solr. response=" + response);
- return null;
- }
- return (List<String>) response.getResponse().get("collections");
- } catch (SolrException e) {
- LOG.error("getCollections() operation failed", e);
- return new ArrayList<>();
- }
- }
-
- private boolean setupCollectionsWithImplicitRouting(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, List<String> allCollectionList)
- throws Exception {
- LOG.info("setupCollectionsWithImplicitRouting(). collectionName=" + solrPropsConfig.getCollection()
- + ", numberOfShards=" + solrPropsConfig.getNumberOfShards());
-
- // Default is true, because if the collection and shard is already there, then it will return true
- boolean returnValue = true;
-
- List<String> shardsList = new ArrayList<String>();
- for (int i = 0; i < solrPropsConfig.getNumberOfShards(); i++) {
- shardsList.add("shard" + i);
- }
- String shardsListStr = StringUtils.join(shardsList, ',');
-
- // Check if collection is already in zookeeper
- if (!allCollectionList.contains(solrPropsConfig.getCollection())) {
- LOG.info("Creating collection " + solrPropsConfig.getCollection() + ", shardsList=" + shardsList);
- CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create();
- collectionCreateRequest.setCollectionName(solrPropsConfig.getCollection());
- collectionCreateRequest.setRouterName("implicit");
- collectionCreateRequest.setShards(shardsListStr);
- collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards());
- collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor());
- collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName());
- collectionCreateRequest.setRouterField(ROUTER_FIELD);
- collectionCreateRequest.setMaxShardsPerNode(solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards());
-
- CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient);
- if (createResponse.getStatus() != 0) {
- returnValue = false;
- LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection()
- + ", shardsList=" + shardsList +", response=" + createResponse);
- } else {
- LOG.info("Created collection " + solrPropsConfig.getCollection() + ", shardsList=" + shardsList);
- }
- } else {
- LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Will check whether it has the required shards");
- Collection<Slice> slices = getSlices(solrClient, solrPropsConfig);
- Collection<String> existingShards = getShards(slices, solrPropsConfig);
- if (existingShards.size() < shardsList.size()) {
- try {
- updateMaximumNumberOfShardsPerCore(slices, solrPropsConfig);
- } catch (Throwable t) {
- returnValue = false;
- LOG.error(String.format("Exception during updating collection (%s)", t));
- }
- }
- for (String shard : shardsList) {
- if (!existingShards.contains(shard)) {
- try {
- LOG.info("Going to add Shard " + shard + " to collection " + solrPropsConfig.getCollection());
- CollectionAdminRequest.CreateShard createShardRequest = new CollectionAdminRequest.CreateShard();
- createShardRequest.setCollectionName(solrPropsConfig.getCollection());
- createShardRequest.setShardName(shard);
- CollectionAdminResponse response = createShardRequest.process(solrClient);
- if (response.getStatus() != 0) {
- LOG.error("Error creating shard " + shard + " in collection " + solrPropsConfig.getCollection() + ", response=" + response);
- returnValue = false;
- break;
- } else {
- LOG.info("Successfully created shard " + shard + " in collection " + solrPropsConfig.getCollection());
- }
- } catch (Throwable t) {
- LOG.error("Error creating shard " + shard + " in collection " + solrPropsConfig.getCollection(), t);
- returnValue = false;
- break;
- }
- }
- }
- }
- return returnValue;
- }
-
- private String getRandomBaseUrl(Collection<Slice> slices) {
- String coreUrl = null;
- if (slices != null) {
- for (Slice slice : slices) {
- if (!slice.getReplicas().isEmpty()) {
- Replica replica = slice.getReplicas().iterator().next();
- coreUrl = replica.getStr("base_url");
- if (coreUrl != null) {
- break;
- }
- }
- }
- }
- return coreUrl;
- }
-
- private void updateMaximumNumberOfShardsPerCore(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) throws IOException {
- String baseUrl = getRandomBaseUrl(slices);
- if (baseUrl != null) {
- CloseableHttpClient httpClient = HttpClientUtil.createClient(null);
- HttpGet request = new HttpGet(baseUrl + String.format(MODIFY_COLLECTION_QUERY,
- solrPropsConfig.getCollection(), MAX_SHARDS_PER_NODE, calculateMaxShardsPerNode(solrPropsConfig)));
- HttpResponse response = httpClient.execute(request);
- if (response.getStatusLine().getStatusCode() != Response.Status.OK.getStatusCode()) {
- throw new IllegalStateException(String.format("Cannot update collection (%s) - increase max number of nodes per core", solrPropsConfig.getCollection()));
- }
- } else {
- throw new IllegalStateException(String.format("Cannot get any core url for updating collection (%s)", solrPropsConfig.getCollection()));
- }
- }
-
- private Collection<Slice> getSlices(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) {
- ZkStateReader reader = solrClient.getZkStateReader();
- return reader.getClusterState().getSlices(solrPropsConfig.getCollection());
- }
-
- private Collection<String> getShards(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) {
- Collection<String> list = new HashSet<>();
- for (Slice slice : slices) {
- for (Replica replica : slice.getReplicas()) {
- LOG.info("colName=" + solrPropsConfig.getCollection() + ", slice.name=" + slice.getName() + ", slice.state=" + slice.getState() +
- ", replica.core=" + replica.getStr("core") + ", replica.state=" + replica.getStr("state"));
- list.add(slice.getName());
- }
- }
- return list;
- }
-
- private boolean createCollection(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, List<String> allCollectionList) throws SolrServerException, IOException {
-
- if (allCollectionList.contains(solrPropsConfig.getCollection())) {
- LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Won't create it");
- return true;
- }
-
- LOG.info("Creating collection " + solrPropsConfig.getCollection() + ", numberOfShards=" + solrPropsConfig.getNumberOfShards() +
- ", replicationFactor=" + solrPropsConfig.getReplicationFactor());
-
- CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create();
- collectionCreateRequest.setCollectionName(solrPropsConfig.getCollection());
- collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards());
- collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor());
- collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName());
- collectionCreateRequest.setMaxShardsPerNode(calculateMaxShardsPerNode(solrPropsConfig));
- CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient);
- if (createResponse.getStatus() != 0) {
- LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection() + ", response=" + createResponse);
- return false;
- } else {
- LOG.info("Created collection " + solrPropsConfig.getCollection() + ", numberOfShards=" + solrPropsConfig.getNumberOfShards() +
- ", replicationFactor=" + solrPropsConfig.getReplicationFactor());
- return true;
- }
- }
-
- private Integer calculateMaxShardsPerNode(SolrPropsConfig solrPropsConfig) {
- return solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards();
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
index 39b65ae..0568fd7 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
@@ -22,6 +22,9 @@ package org.apache.ambari.logsearch.dao;
import org.apache.ambari.logsearch.common.LogSearchContext;
import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.conf.SolrKerberosConfig;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
import org.apache.ambari.logsearch.util.RESTErrorUtil;
import org.apache.ambari.logsearch.util.SolrUtil;
import org.apache.log4j.Logger;
@@ -36,6 +39,7 @@ import org.springframework.data.solr.core.SolrCallback;
import org.springframework.data.solr.core.SolrTemplate;
import org.springframework.data.solr.core.query.SolrDataQuery;
+import javax.inject.Inject;
import java.io.IOException;
public abstract class SolrDaoBase {
@@ -44,6 +48,9 @@ public abstract class SolrDaoBase {
private static final Logger LOG_PERFORMANCE = Logger.getLogger("org.apache.ambari.logsearch.performance");
private LogType logType;
+
+ @Inject
+ private SolrKerberosConfig solrKerberosConfig;
protected SolrDaoBase(LogType logType) {
this.logType = logType;
@@ -108,5 +115,14 @@ public abstract class SolrDaoBase {
public abstract SolrTemplate getSolrTemplate();
- public abstract SolrSchemaFieldDao getSolrSchemaFieldDao();
+ public abstract void setSolrTemplate(SolrTemplate solrTemplate);
+
+ public abstract SolrCollectionState getSolrCollectionState();
+
+ public abstract SolrPropsConfig getSolrPropsConfig();
+
+ public SolrKerberosConfig getSolrKerberosConfig() {
+ return this.solrKerberosConfig;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
index e6a562e..01378c1 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
@@ -43,7 +43,6 @@ import org.apache.solr.common.util.NamedList;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.data.solr.core.SolrTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import java.io.IOException;
@@ -54,54 +53,30 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import javax.annotation.PostConstruct;
import javax.inject.Inject;
-import javax.inject.Named;
public class SolrSchemaFieldDao {
private static final Logger LOG = LoggerFactory.getLogger(SolrSchemaFieldDao.class);
private static final int RETRY_SECOND = 30;
-
+
@Inject
- @Named("serviceSolrTemplate")
- private SolrTemplate serviceSolrTemplate;
+ private ServiceLogsSolrDao serviceLogsSolrDao;
@Inject
- @Named("auditSolrTemplate")
- private SolrTemplate auditSolrTemplate;
+ private AuditSolrDao auditSolrDao;
@Inject
private SolrUserPropsConfig solrUserConfigPropsConfig;
- private CloudSolrClient serviceSolrClient;
- private CloudSolrClient auditSolrClient;
-
private int retryCount;
private int skipCount;
-
- private boolean serviceCollectionSetUp = false;
- private boolean auditCollectionSetUp = false;
private Map<String, String> serviceSchemaFieldNameMap = new HashMap<>();
private Map<String, String> serviceSchemaFieldTypeMap = new HashMap<>();
private Map<String, String> auditSchemaFieldNameMap = new HashMap<>();
private Map<String, String> auditSchemaFieldTypeMap = new HashMap<>();
-
- @PostConstruct
- public void init() {
- this.serviceSolrClient = (CloudSolrClient) serviceSolrTemplate.getSolrClient();
- this.auditSolrClient = (CloudSolrClient) auditSolrTemplate.getSolrClient();
- }
-
- void serviceCollectionSetUp() {
- this.serviceCollectionSetUp = true;
- }
-
- void auditCollectionSetUp() {
- this.auditCollectionSetUp = true;
- }
@Scheduled(fixedDelay = RETRY_SECOND * 1000)
public void populateAllSchemaFields() {
@@ -109,11 +84,12 @@ public class SolrSchemaFieldDao {
skipCount--;
return;
}
-
- if (serviceCollectionSetUp) {
+ if (serviceLogsSolrDao.getSolrCollectionState().isSolrCollectionReady()) {
+ CloudSolrClient serviceSolrClient = (CloudSolrClient) serviceLogsSolrDao.getSolrTemplate().getSolrClient();
populateSchemaFields(serviceSolrClient, serviceSchemaFieldNameMap, serviceSchemaFieldTypeMap);
}
- if (auditCollectionSetUp) {
+ if (auditSolrDao.getSolrCollectionState().isSolrCollectionReady()) {
+ CloudSolrClient auditSolrClient = (CloudSolrClient) auditSolrDao.getSolrTemplate().getSolrClient();
populateSchemaFields(auditSolrClient, auditSchemaFieldNameMap, auditSchemaFieldTypeMap);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
index 106bedc..13d7e03 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
@@ -32,7 +32,11 @@ import org.apache.ambari.logsearch.common.HadoopServiceConfigHelper;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.common.LogSearchContext;
import org.apache.ambari.logsearch.common.LogType;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
import org.apache.ambari.logsearch.conf.SolrUserPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.configurer.LogfeederFilterConfigurer;
+import org.apache.ambari.logsearch.configurer.SolrCollectionConfigurer;
import org.apache.ambari.logsearch.model.common.LogFeederDataMap;
import org.apache.ambari.logsearch.model.common.LogfeederFilterData;
import org.apache.solr.client.solrj.SolrQuery;
@@ -66,12 +70,13 @@ public class UserConfigSolrDao extends SolrDaoBase {
private SolrUserPropsConfig solrUserConfig;
@Inject
- private SolrCollectionDao solrCollectionDao;
-
- @Inject
@Named("userConfigSolrTemplate")
private SolrTemplate userConfigSolrTemplate;
+ @Inject
+ @Named("solrUserConfigState")
+ private SolrCollectionState solrUserConfigState;
+
public UserConfigSolrDao() {
super(LogType.SERVICE);
}
@@ -81,6 +86,11 @@ public class UserConfigSolrDao extends SolrDaoBase {
return userConfigSolrTemplate;
}
+ @Override
+ public void setSolrTemplate(SolrTemplate solrTemplate) {
+ this.userConfigSolrTemplate = solrTemplate;
+ }
+
@PostConstruct
public void postConstructor() {
String solrUrl = solrUserConfig.getSolrUrl();
@@ -88,24 +98,14 @@ public class UserConfigSolrDao extends SolrDaoBase {
String collection = solrUserConfig.getCollection();
try {
- solrCollectionDao.checkSolrStatus(getSolrClient());
- solrCollectionDao.setupCollections(getSolrClient(), solrUserConfig);
- intializeLogFeederFilter();
-
+ new SolrCollectionConfigurer(this).start();
+ new LogfeederFilterConfigurer(this).start();
} catch (Exception e) {
LOG.error("error while connecting to Solr for history logs : solrUrl=" + solrUrl + ", zkConnectString=" + zkConnectString +
", collection=" + collection, e);
}
}
- private void intializeLogFeederFilter() {
- try {
- getUserFilter();
- } catch (SolrServerException | IOException e) {
- LOG.error("not able to save logfeeder filter while initialization", e);
- }
- }
-
public void saveUserFilter(LogFeederDataMap logfeederFilterWrapper) throws SolrException, SolrServerException, IOException {
String filterName = LogSearchConstants.LOGFEEDER_FILTER_NAME;
String json = JSONUtil.objToJson(logfeederFilterWrapper);
@@ -179,7 +179,12 @@ public class UserConfigSolrDao extends SolrDaoBase {
}
@Override
- public SolrSchemaFieldDao getSolrSchemaFieldDao() {
- throw new UnsupportedOperationException();
+ public SolrCollectionState getSolrCollectionState() {
+ return solrUserConfigState;
+ }
+
+ @Override
+ public SolrPropsConfig getSolrPropsConfig() {
+ return solrUserConfig;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
index 0fc9852..caf0636 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
@@ -115,4 +115,11 @@ public class DocConstants {
public static final String GET_USER_FILTER_BY_ID_OD = "Get user filter by id";
public static final String GET_ALL_USER_NAMES_OD = "Get all user names";
}
+
+ public class StatusOperationDescriptions {
+ public static final String STATUS_OD = "Get statuses for collections (not health state - show true if something already done)";
+ public static final String SERVICE_LOGS_STATUS_OD = "Get statuses for service log collection (not health state - show true if something already done)";
+ public static final String AUDIT_LOGS_STATUS_OD = "Get statuses for collections (not health state - show true if something already done)";
+ public static final String USER_CONFIG_STATUS_OD = "Get statuses for userconfig collection (not health state - show true if something already done)";
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ACLHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ACLHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ACLHandler.java
new file mode 100644
index 0000000..fde176f
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ACLHandler.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.handler;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.SolrZooKeeper;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+public class ACLHandler implements SolrZkRequestHandler<Boolean> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ACLHandler.class);
+
+ @Override
+ public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception {
+ List<ACL> aclsToSetList = solrPropsConfig.getZkAcls();
+ if (CollectionUtils.isNotEmpty(aclsToSetList)) {
+ LOG.info("Setting acls for '{}' collection...", solrPropsConfig.getCollection());
+ SolrZkClient zkClient = solrClient.getZkStateReader().getZkClient();
+ SolrZooKeeper solrZooKeeper = zkClient.getSolrZooKeeper();
+ String collectionPath = String.format("/collections/%s", solrPropsConfig.getCollection());
+ String configsPath = String.format("/configs/%s", solrPropsConfig.getConfigName());
+ List<ACL> collectionAcls = solrZooKeeper.getACL(collectionPath, new Stat());
+ if (isRefreshAclsNeeded(aclsToSetList, collectionAcls)) {
+ LOG.info("Acls differs for {}, update acls.", collectionPath);
+ setRecursivelyOn(solrZooKeeper, collectionPath, aclsToSetList);
+ }
+ List<ACL> configsAcls = solrZooKeeper.getACL(configsPath, new Stat());
+ if (isRefreshAclsNeeded(aclsToSetList, configsAcls)) {
+ LOG.info("Acls differs for {}, update acls.", configsPath);
+ setRecursivelyOn(solrZooKeeper, configsPath, aclsToSetList);
+ }
+ }
+ return true;
+ }
+
+ private boolean isRefreshAclsNeeded(List<ACL> acls, List<ACL> newAcls) {
+ boolean result = false;
+ if (acls != null) {
+ if (acls.size() != newAcls.size()) {
+ return true;
+ }
+ result = aclDiffers(acls, newAcls);
+ if (!result) {
+ result = aclDiffers(newAcls, acls);
+ }
+ }
+ return result;
+ }
+
+ private boolean aclDiffers(List<ACL> aclList1, List<ACL> aclList2) {
+ for (ACL acl : aclList1) {
+ for (ACL newAcl : aclList2) {
+ if (acl.getId() != null && acl.getId().getId().equals(newAcl.getId().getId())
+ && acl.getPerms() != newAcl.getPerms()) {
+ LOG.info("ACL for '{}' differs: '{}' on znode, should be '{}'",
+ acl.getId().getId(), acl.getPerms(), newAcl.getPerms());
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private void setRecursivelyOn(SolrZooKeeper solrZooKeeper, String node, List<ACL> acls)
+ throws KeeperException, InterruptedException {
+ solrZooKeeper.setACL(node, acls, -1);
+ for (String child : solrZooKeeper.getChildren(node, null)) {
+ String path = node.endsWith("/") ? node + child : node + "/" + child;
+ setRecursivelyOn(solrZooKeeper, path, acls);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
new file mode 100644
index 0000000..752a1e1
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.handler;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+
+import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.ROUTER_FIELD;
+
+public class CreateCollectionHandler implements SolrZkRequestHandler<Boolean> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(CreateCollectionHandler.class);
+
+ private static final String MODIFY_COLLECTION_QUERY = "/admin/collections?action=MODIFYCOLLECTION&collection=%s&%s=%d";
+ private static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode";
+
+ private List<String> allCollectionList;
+
+ public CreateCollectionHandler(List<String> allCollectionList) {
+ this.allCollectionList = allCollectionList;
+ }
+
+ @Override
+ public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception {
+ boolean result;
+ if (solrPropsConfig.getSplitInterval().equalsIgnoreCase("none")) {
+ result = createCollection(solrClient, solrPropsConfig, this.allCollectionList);
+ } else {
+ result = setupCollectionsWithImplicitRouting(solrClient, solrPropsConfig, this.allCollectionList);
+ }
+ return result;
+ }
+
+ private boolean setupCollectionsWithImplicitRouting(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, List<String> allCollectionList)
+ throws Exception {
+ LOG.info("setupCollectionsWithImplicitRouting(). collectionName=" + solrPropsConfig.getCollection()
+ + ", numberOfShards=" + solrPropsConfig.getNumberOfShards());
+
+ // Default is true, because if the collection and shard is already there, then it will return true
+ boolean returnValue = true;
+
+ List<String> shardsList = new ArrayList<String>();
+ for (int i = 0; i < solrPropsConfig.getNumberOfShards(); i++) {
+ shardsList.add("shard" + i);
+ }
+ String shardsListStr = StringUtils.join(shardsList, ',');
+
+ // Check if collection is already in zookeeper
+ if (!allCollectionList.contains(solrPropsConfig.getCollection())) {
+ LOG.info("Creating collection " + solrPropsConfig.getCollection() + ", shardsList=" + shardsList);
+ CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create();
+ collectionCreateRequest.setCollectionName(solrPropsConfig.getCollection());
+ collectionCreateRequest.setRouterName("implicit");
+ collectionCreateRequest.setShards(shardsListStr);
+ collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards());
+ collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor());
+ collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName());
+ collectionCreateRequest.setRouterField(ROUTER_FIELD);
+ collectionCreateRequest.setMaxShardsPerNode(solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards());
+
+ CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient);
+ if (createResponse.getStatus() != 0) {
+ returnValue = false;
+ LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection()
+ + ", shardsList=" + shardsList +", response=" + createResponse);
+ } else {
+ LOG.info("Created collection " + solrPropsConfig.getCollection() + ", shardsList=" + shardsList);
+ }
+ } else {
+ LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Will check whether it has the required shards");
+ Collection<Slice> slices = getSlices(solrClient, solrPropsConfig);
+ Collection<String> existingShards = getShards(slices, solrPropsConfig);
+ if (existingShards.size() < shardsList.size()) {
+ try {
+ updateMaximumNumberOfShardsPerCore(slices, solrPropsConfig);
+ } catch (Throwable t) {
+ returnValue = false;
+ LOG.error(String.format("Exception during updating collection (%s)", t));
+ }
+ }
+ for (String shard : shardsList) {
+ if (!existingShards.contains(shard)) {
+ try {
+ LOG.info("Going to add Shard " + shard + " to collection " + solrPropsConfig.getCollection());
+ CollectionAdminRequest.CreateShard createShardRequest = new CollectionAdminRequest.CreateShard();
+ createShardRequest.setCollectionName(solrPropsConfig.getCollection());
+ createShardRequest.setShardName(shard);
+ CollectionAdminResponse response = createShardRequest.process(solrClient);
+ if (response.getStatus() != 0) {
+ LOG.error("Error creating shard " + shard + " in collection " + solrPropsConfig.getCollection() + ", response=" + response);
+ returnValue = false;
+ break;
+ } else {
+ LOG.info("Successfully created shard " + shard + " in collection " + solrPropsConfig.getCollection());
+ }
+ } catch (Throwable t) {
+ LOG.error("Error creating shard " + shard + " in collection " + solrPropsConfig.getCollection(), t);
+ returnValue = false;
+ break;
+ }
+ }
+ }
+ }
+ return returnValue;
+ }
+
+ private boolean createCollection(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, List<String> allCollectionList) throws SolrServerException, IOException {
+
+ if (allCollectionList.contains(solrPropsConfig.getCollection())) {
+ LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Won't create it");
+ return true;
+ }
+
+ LOG.info("Creating collection " + solrPropsConfig.getCollection() + ", numberOfShards=" + solrPropsConfig.getNumberOfShards() +
+ ", replicationFactor=" + solrPropsConfig.getReplicationFactor());
+
+ CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create();
+ collectionCreateRequest.setCollectionName(solrPropsConfig.getCollection());
+ collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards());
+ collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor());
+ collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName());
+ collectionCreateRequest.setMaxShardsPerNode(calculateMaxShardsPerNode(solrPropsConfig));
+ CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient);
+ if (createResponse.getStatus() != 0) {
+ LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection() + ", response=" + createResponse);
+ return false;
+ } else {
+ LOG.info("Created collection " + solrPropsConfig.getCollection() + ", numberOfShards=" + solrPropsConfig.getNumberOfShards() +
+ ", replicationFactor=" + solrPropsConfig.getReplicationFactor());
+ return true;
+ }
+ }
+
+ private void updateMaximumNumberOfShardsPerCore(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) throws IOException {
+ String baseUrl = getRandomBaseUrl(slices);
+ if (baseUrl != null) {
+ CloseableHttpClient httpClient = HttpClientUtil.createClient(null);
+ HttpGet request = new HttpGet(baseUrl + String.format(MODIFY_COLLECTION_QUERY,
+ solrPropsConfig.getCollection(), MAX_SHARDS_PER_NODE, calculateMaxShardsPerNode(solrPropsConfig)));
+ HttpResponse response = httpClient.execute(request);
+ if (response.getStatusLine().getStatusCode() != Response.Status.OK.getStatusCode()) {
+ throw new IllegalStateException(String.format("Cannot update collection (%s) - increase max number of nodes per core", solrPropsConfig.getCollection()));
+ }
+ } else {
+ throw new IllegalStateException(String.format("Cannot get any core url for updating collection (%s)", solrPropsConfig.getCollection()));
+ }
+ }
+
+ private Collection<Slice> getSlices(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) {
+ ZkStateReader reader = solrClient.getZkStateReader();
+ return reader.getClusterState().getSlices(solrPropsConfig.getCollection());
+ }
+
+ private Collection<String> getShards(Collection<Slice> slices, SolrPropsConfig solrPropsConfig) {
+ Collection<String> list = new HashSet<>();
+ for (Slice slice : slices) {
+ for (Replica replica : slice.getReplicas()) {
+ LOG.info("colName=" + solrPropsConfig.getCollection() + ", slice.name=" + slice.getName() + ", slice.state=" + slice.getState() +
+ ", replica.core=" + replica.getStr("core") + ", replica.state=" + replica.getStr("state"));
+ list.add(slice.getName());
+ }
+ }
+ return list;
+ }
+
+ private String getRandomBaseUrl(Collection<Slice> slices) {
+ String coreUrl = null;
+ if (slices != null) {
+ for (Slice slice : slices) {
+ if (!slice.getReplicas().isEmpty()) {
+ Replica replica = slice.getReplicas().iterator().next();
+ coreUrl = replica.getStr("base_url");
+ if (coreUrl != null) {
+ break;
+ }
+ }
+ }
+ }
+ return coreUrl;
+ }
+
+ private Integer calculateMaxShardsPerNode(SolrPropsConfig solrPropsConfig) {
+ return solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ListCollectionHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ListCollectionHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ListCollectionHandler.java
new file mode 100644
index 0000000..124ce40
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ListCollectionHandler.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.handler;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.common.SolrException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ListCollectionHandler implements SolrZkRequestHandler<List<String>> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ListCollectionHandler.class);
+
+ @Override
+ public List<String> handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception {
+ try {
+ CollectionAdminRequest.List colListReq = new CollectionAdminRequest.List();
+ CollectionAdminResponse response = colListReq.process(solrClient);
+ if (response.getStatus() != 0) {
+ LOG.error("Error getting collection list from solr. response=" + response);
+ return null;
+ }
+ return (List<String>) response.getResponse().get("collections");
+ } catch (SolrException e) {
+ LOG.error("getCollections() operation failed", e);
+ return new ArrayList<>();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ReloadCollectionHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ReloadCollectionHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ReloadCollectionHandler.java
new file mode 100644
index 0000000..52f3366
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/ReloadCollectionHandler.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.handler;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ReloadCollectionHandler implements SolrZkRequestHandler<Boolean> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ReloadCollectionHandler.class);
+
+ @Override
+ public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception {
+ boolean result = false;
+ try {
+ LOG.info("Reload collection - '{}'", solrPropsConfig.getCollection());
+ CollectionAdminRequest.Reload reloadCollectionRequest = new CollectionAdminRequest.Reload();
+ reloadCollectionRequest.setCollectionName(solrPropsConfig.getCollection());
+ reloadCollectionRequest.process(solrClient);
+ result = true;
+ } catch (Exception e) {
+ LOG.error(String.format("Reload collection ('%s') failed.", solrPropsConfig.getCollection()), e);
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/SolrZkRequestHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/SolrZkRequestHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/SolrZkRequestHandler.java
new file mode 100644
index 0000000..85ae6cb
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/SolrZkRequestHandler.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.handler;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+
+interface SolrZkRequestHandler<T> {
+ T handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java
new file mode 100644
index 0000000..23defea
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.handler;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.commons.io.FileUtils;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkConfigManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Paths;
+import java.util.UUID;
+
+public class UploadConfigurationHandler implements SolrZkRequestHandler<Boolean> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(UploadConfigurationHandler.class);
+
+ private static final String SCHEMA_FILE = "managed-schema";
+ private static final String SOLR_CONFIG_FILE = "solrconfig.xml";
+
+ private File configSetFolder;
+
+ public UploadConfigurationHandler(File configSetFolder) {
+ this.configSetFolder = configSetFolder;
+ }
+
+ @Override
+ public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception {
+ boolean reloadCollectionNeeded = false;
+ String separator = FileSystems.getDefault().getSeparator();
+ String downloadFolderLocation = String.format("%s%s%s%s%s", System.getProperty("java.io.tmpdir"), separator,
+ UUID.randomUUID().toString(), separator, solrPropsConfig.getConfigName());
+ solrClient.connect();
+ SolrZkClient zkClient = solrClient.getZkStateReader().getZkClient();
+ File tmpDir = new File(downloadFolderLocation);
+ try {
+ ZkConfigManager zkConfigManager = new ZkConfigManager(zkClient);
+ boolean configExists = zkConfigManager.configExists(solrPropsConfig.getConfigName());
+ if (configExists) {
+ LOG.info("Config set exists for '{}' collection. Refreshing it if needed...", solrPropsConfig.getCollection());
+ if (!tmpDir.mkdirs()) {
+ LOG.error("Cannot create directories for '{}'", tmpDir.getAbsolutePath());
+ }
+ zkConfigManager.downloadConfigDir(solrPropsConfig.getConfigName(), Paths.get(downloadFolderLocation));
+ File[] listOfFiles = configSetFolder.listFiles();
+ if (listOfFiles != null) {
+ for (File file : listOfFiles) {
+ if (file.getName().equals(SOLR_CONFIG_FILE) || file.getName().equals(SCHEMA_FILE)) { // TODO: try to find an another solution to reload schema
+ if (!FileUtils.contentEquals(file, new File(String.format("%s%s%s", downloadFolderLocation, separator, file.getName())))){
+ LOG.info("One of the local solr config file differs ('{}'), upload config set to zookeeper", file.getName());
+ zkConfigManager.uploadConfigDir(configSetFolder.toPath(), solrPropsConfig.getConfigName());
+ reloadCollectionNeeded = true;
+ break;
+ }
+ }
+ }
+ }
+ } else {
+ LOG.info("Config set does not exist for '{}' collection. Uploading it to zookeeper...", solrPropsConfig.getCollection());
+ File[] listOfFiles = configSetFolder.listFiles();
+ if (listOfFiles != null) {
+ zkConfigManager.uploadConfigDir(configSetFolder.toPath(), solrPropsConfig.getConfigName());
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(String.format("Cannot upload configurations to zk. (collection: %s, config set folder: %s)",
+ solrPropsConfig.getCollection(), solrPropsConfig.getConfigSetFolder()), e);
+ } finally {
+ if (tmpDir.exists()) {
+ try {
+ FileUtils.deleteDirectory(tmpDir);
+ } catch (IOException e){
+ LOG.error("Cannot delete temp directory.", e);
+ }
+ }
+ }
+ return reloadCollectionNeeded;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/StatusResource.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/StatusResource.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/StatusResource.java
new file mode 100644
index 0000000..8cc6b94
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/StatusResource.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.rest;
+
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.springframework.context.annotation.Scope;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.ambari.logsearch.doc.DocConstants.StatusOperationDescriptions.AUDIT_LOGS_STATUS_OD;
+import static org.apache.ambari.logsearch.doc.DocConstants.StatusOperationDescriptions.SERVICE_LOGS_STATUS_OD;
+import static org.apache.ambari.logsearch.doc.DocConstants.StatusOperationDescriptions.STATUS_OD;
+import static org.apache.ambari.logsearch.doc.DocConstants.StatusOperationDescriptions.USER_CONFIG_STATUS_OD;
+
+@Api(value = "status", description = "Status Operations")
+@Path("status")
+@Named
+@Scope("request")
+public class StatusResource {
+
+ @Inject
+ @Named("solrServiceLogsState")
+ private SolrCollectionState solrServiceLogsState;
+
+ @Inject
+ @Named("solrAuditLogsState")
+ private SolrCollectionState solrAuditLogsState;
+
+ @Inject
+ @Named("solrUserConfigState")
+ private SolrCollectionState solrUserConfigState;
+
+ @GET
+ @Produces({"application/json"})
+ @ApiOperation(STATUS_OD)
+ public Map<String, SolrCollectionState> getStatus() {
+ Map<String, SolrCollectionState> response = new HashMap<>();
+ response.put("serviceLogs", solrServiceLogsState);
+ response.put("auditLogs", solrAuditLogsState);
+ response.put("userConfig", solrUserConfigState);
+ return response;
+ }
+
+ @GET
+ @Path("/servicelogs")
+ @Produces({"application/json"})
+ @ApiOperation(SERVICE_LOGS_STATUS_OD)
+ public SolrCollectionState getServiceLogStatus() {
+ return solrServiceLogsState;
+ }
+
+ @GET
+ @Path("/auditlogs")
+ @Produces({"application/json"})
+ @ApiOperation(AUDIT_LOGS_STATUS_OD)
+ public SolrCollectionState getSolrAuditLogsStatus() {
+ return solrAuditLogsState;
+ }
+
+ @GET
+ @Path("/userconfig")
+ @Produces({"application/json"})
+ @ApiOperation(USER_CONFIG_STATUS_OD)
+ public SolrCollectionState getSolrUserConfigStatus() {
+ return solrUserConfigState;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java
index 532428b..b1a53f2 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/RESTErrorUtil.java
@@ -41,14 +41,18 @@ public class RESTErrorUtil {
return createRESTException(response, HttpServletResponse.SC_BAD_REQUEST);
}
- public static WebApplicationException createRESTException(String errorMessage, MessageEnums messageEnum) {
- List<MessageData> messageList = new ArrayList<MessageData>();
+ public static VResponse createMessageResponse(String errorMessage, MessageEnums messageEnum) {
+ List<MessageData> messageList = new ArrayList<>();
messageList.add(messageEnum.getMessage());
-
VResponse response = new VResponse();
response.setStatusCode(VResponse.STATUS_ERROR);
response.setMsgDesc(errorMessage);
response.setMessageList(messageList);
+ return response;
+ }
+
+ public static WebApplicationException createRESTException(String errorMessage, MessageEnums messageEnum) {
+ VResponse response = createMessageResponse(errorMessage, messageEnum);
WebApplicationException webAppEx = createRESTException(response);
logger.error("Operation error. response=" + response, webAppEx);
return webAppEx;
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/AbstractLogsearchGlobalStateFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/AbstractLogsearchGlobalStateFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/AbstractLogsearchGlobalStateFilter.java
new file mode 100644
index 0000000..4d529fb
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/AbstractLogsearchGlobalStateFilter.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.web.filters;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.common.VResponse;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.security.web.util.matcher.RequestMatcher;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+
+abstract public class AbstractLogsearchGlobalStateFilter implements Filter {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractLogsearchGlobalStateFilter.class);
+
+ protected static final String ZNODE_NOT_READY_MSG = "ZNode is not available for %s. (connection string: %s, endpoint: %s)";
+ protected static final String ZK_CONFIG_NOT_READY_MSG = "Collection configuration has not uploaded yet for %s. (configuration name: %s, collection name: %s, endpoint: %s)";
+ protected static final String SOLR_COLLECTION_NOT_READY_MSG = "Solr has not accessible yet for %s collection. (endpoint: %s)";
+
+ private SolrCollectionState solrCollectionState;
+ private RequestMatcher requestMatcher;
+ private SolrPropsConfig solrPropsConfig;
+
+ public AbstractLogsearchGlobalStateFilter(RequestMatcher requestMatcher, SolrCollectionState state, SolrPropsConfig solrPropsConfig) {
+ this.requestMatcher = requestMatcher;
+ this.solrCollectionState = state;
+ this.solrPropsConfig = solrPropsConfig;
+ }
+
+ @Override
+ public void init(FilterConfig filterConfig) throws ServletException {
+ }
+
+ @Override
+ public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException {
+ HttpServletRequest request = (HttpServletRequest) servletRequest;
+ if (requestMatcher.matches(request)) {
+ VResponse errorResponse = getErrorResponse(solrCollectionState, solrPropsConfig, request);
+ if (errorResponse != null) {
+ LOG.info("{} request is filtered out: {}", request.getRequestURL(), errorResponse.getMsgDesc());
+ HttpServletResponse resp = (HttpServletResponse) servletResponse;
+ resp.setStatus(500);
+ resp.setContentType("application/json");
+ resp.getWriter().print(createStringFromErrorMessageObject(errorResponse));
+ return;
+ }
+ }
+ filterChain.doFilter(servletRequest, servletResponse);
+ }
+
+ @Override
+ public void destroy() {
+ }
+
+ /**
+ * Fill the error message with data in case of any condition based on collection state.
+ * If return value is null, that means there were no any errors
+ * @param solrCollectionState object to store solr state (e.g. : zookeeper/solr collection availability)
+ * @param solrPropsConfig object to store details for solr collection (e.g. : collection name, connection string)
+ */
+ public abstract VResponse getErrorResponse(SolrCollectionState solrCollectionState, SolrPropsConfig solrPropsConfig, HttpServletRequest request);
+
+ private String createStringFromErrorMessageObject(VResponse responseObject) {
+ try {
+ ObjectMapper mapper = new ObjectMapper();
+ return mapper.writeValueAsString(responseObject);
+ } catch (Exception e) {
+ throw RESTErrorUtil.createRESTException("Cannot parse response object on backend", MessageEnums.ERROR_CREATING_OBJECT);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuditLogsStateFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuditLogsStateFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuditLogsStateFilter.java
new file mode 100644
index 0000000..117fdd4
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuditLogsStateFilter.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.web.filters;
+
+import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.common.VResponse;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.springframework.security.web.util.matcher.RequestMatcher;
+
+import javax.servlet.http.HttpServletRequest;
+
+public class LogsearchAuditLogsStateFilter extends AbstractLogsearchGlobalStateFilter {
+
+ public LogsearchAuditLogsStateFilter(RequestMatcher requestMatcher, SolrCollectionState state, SolrPropsConfig solrPropsConfig) {
+ super(requestMatcher, state, solrPropsConfig);
+ }
+
+ @Override
+ public VResponse getErrorResponse(SolrCollectionState solrCollectionState, SolrPropsConfig solrPropsConfig, HttpServletRequest request) {
+ String requestUri = request.getRequestURI();
+ if (!solrCollectionState.isZnodeReady()) {
+ return RESTErrorUtil.createMessageResponse(String.format(ZNODE_NOT_READY_MSG,
+ "audit logs", solrPropsConfig.getZkConnectString(), requestUri), MessageEnums.ZNODE_NOT_READY);
+ } else if (!solrCollectionState.isConfigurationUploaded()) {
+ return RESTErrorUtil.createMessageResponse(String.format(ZK_CONFIG_NOT_READY_MSG, "audit logs",
+ solrPropsConfig.getConfigName(), solrPropsConfig.getCollection(), requestUri), MessageEnums.ZK_CONFIG_NOT_READY);
+ } else if (!solrCollectionState.isSolrCollectionReady()) {
+ return RESTErrorUtil.createMessageResponse(String.format(SOLR_COLLECTION_NOT_READY_MSG,
+ solrPropsConfig.getCollection(), requestUri), MessageEnums.SOLR_COLLECTION_NOT_READY);
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchServiceLogsStateFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchServiceLogsStateFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchServiceLogsStateFilter.java
new file mode 100644
index 0000000..74b30e1
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchServiceLogsStateFilter.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.web.filters;
+
+import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.common.VResponse;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.springframework.security.web.util.matcher.RequestMatcher;
+
+import javax.servlet.http.HttpServletRequest;
+
+public class LogsearchServiceLogsStateFilter extends AbstractLogsearchGlobalStateFilter {
+
+ public LogsearchServiceLogsStateFilter(RequestMatcher requestMatcher, SolrCollectionState state, SolrPropsConfig solrPropsConfig) {
+ super(requestMatcher, state, solrPropsConfig);
+ }
+
+ @Override
+ public VResponse getErrorResponse(SolrCollectionState solrCollectionState, SolrPropsConfig solrPropsConfig, HttpServletRequest request) {
+ String requestUri = request.getRequestURI();
+ if (!solrCollectionState.isZnodeReady()) {
+ return RESTErrorUtil.createMessageResponse(String.format(ZNODE_NOT_READY_MSG,
+ "service logs", solrPropsConfig.getZkConnectString(), requestUri), MessageEnums.ZNODE_NOT_READY);
+ } else if (!solrCollectionState.isConfigurationUploaded()) {
+ return RESTErrorUtil.createMessageResponse(String.format(ZK_CONFIG_NOT_READY_MSG, "service logs",
+ solrPropsConfig.getConfigName(), solrPropsConfig.getCollection(), requestUri), MessageEnums.ZK_CONFIG_NOT_READY);
+ } else if (!solrCollectionState.isSolrCollectionReady()) {
+ return RESTErrorUtil.createMessageResponse(String.format(SOLR_COLLECTION_NOT_READY_MSG,
+ solrPropsConfig.getCollection(), requestUri), MessageEnums.SOLR_COLLECTION_NOT_READY);
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUserConfigStateFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUserConfigStateFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUserConfigStateFilter.java
new file mode 100644
index 0000000..037bed0
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUserConfigStateFilter.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.web.filters;
+
+import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.common.VResponse;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.springframework.security.web.util.matcher.RequestMatcher;
+
+import javax.servlet.http.HttpServletRequest;
+
+public class LogsearchUserConfigStateFilter extends AbstractLogsearchGlobalStateFilter {
+
+
+ public LogsearchUserConfigStateFilter(RequestMatcher requestMatcher, SolrCollectionState state, SolrPropsConfig solrPropsConfig) {
+ super(requestMatcher, state, solrPropsConfig);
+ }
+
+ @Override
+ public VResponse getErrorResponse(SolrCollectionState solrCollectionState, SolrPropsConfig solrPropsConfig, HttpServletRequest request) {
+ String requestUri = request.getRequestURI();
+ if (!solrCollectionState.isZnodeReady()) {
+ return RESTErrorUtil.createMessageResponse(String.format(ZNODE_NOT_READY_MSG,
+ "userconfig", solrPropsConfig.getZkConnectString(), requestUri), MessageEnums.ZNODE_NOT_READY);
+ } else if (!solrCollectionState.isConfigurationUploaded()) {
+ return RESTErrorUtil.createMessageResponse(String.format(ZK_CONFIG_NOT_READY_MSG, "userconfig",
+ solrPropsConfig.getConfigName(), solrPropsConfig.getCollection(), requestUri), MessageEnums.ZK_CONFIG_NOT_READY);
+ } else if (!solrCollectionState.isSolrCollectionReady()) {
+ return RESTErrorUtil.createMessageResponse(String.format(SOLR_COLLECTION_NOT_READY_MSG,
+ solrPropsConfig.getCollection(), requestUri), MessageEnums.SOLR_COLLECTION_NOT_READY);
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js
index 85c8245..15322ef 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Utils.js
@@ -411,8 +411,15 @@ define(['require',
try {
if (!errorShown) {
errorShown = true;
+ var errorMessage = "Some issues on server, Please try again later."
+ if (error != null && error.responseText != null) {
+ var errorObj = JSON.parse(error.responseText);
+ if (errorObj.hasOwnProperty('msgDesc')) {
+ errorMessage = errorObj.msgDesc;
+ }
+ }
Utils.notifyError({
- content: "Some issue on server, Please try again later."
+ content: errorMessage
});
setTimeout(function() {
errorShown = false;
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java
index fa6162a..6bb5a77 100644
--- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java
+++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/AmbariSolrCloudClient.java
@@ -20,7 +20,6 @@ package org.apache.ambari.logsearch.solr;
import org.apache.ambari.logsearch.solr.commands.CheckConfigZkCommand;
import org.apache.ambari.logsearch.solr.commands.CreateCollectionCommand;
-import org.apache.ambari.logsearch.solr.commands.CreateSaslUsersZkCommand;
import org.apache.ambari.logsearch.solr.commands.CreateShardCommand;
import org.apache.ambari.logsearch.solr.commands.CreateSolrZnodeZkCommand;
import org.apache.ambari.logsearch.solr.commands.DownloadConfigZkCommand;
@@ -34,7 +33,6 @@ import org.apache.ambari.logsearch.solr.commands.SetClusterPropertyZkCommand;
import org.apache.ambari.logsearch.solr.commands.UploadConfigZkCommand;
import org.apache.ambari.logsearch.solr.commands.CheckZnodeZkCommand;
import org.apache.ambari.logsearch.solr.util.ShardUtils;
-import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
@@ -129,18 +127,6 @@ public class AmbariSolrCloudClient {
}
/**
- * Add sasl user (only if not exist on the znode).
- */
- public String addSaslUsers() throws Exception {
- LOG.info("Add sasl user to znode: {}", this.saslUsers);
- String newUsers = new CreateSaslUsersZkCommand(getRetryTimes(), getInterval()).run(this);
- if (StringUtils.isNotEmpty(newUsers)) {
- LOG.info("New sasl users added to znode: {}", newUsers);
- }
- return newUsers;
- }
-
- /**
* Create a znode only if it does not exist. Return 0 code if it exists.
*/
public void createZnode() throws Exception {
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java
deleted file mode 100644
index b144b41..0000000
--- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/CreateSaslUsersZkCommand.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logsearch.solr.commands;
-
-import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient;
-import org.apache.ambari.logsearch.solr.util.AclUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.cloud.SolrZooKeeper;
-import org.apache.zookeeper.ZooDefs;
-import org.apache.zookeeper.data.ACL;
-import org.apache.zookeeper.data.Id;
-import org.apache.zookeeper.data.Stat;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-
-public class CreateSaslUsersZkCommand extends AbstractZookeeperRetryCommand<String> {
-
- public CreateSaslUsersZkCommand(int maxRetries, int interval) {
- super(maxRetries, interval);
- }
-
- @Override
- protected String executeZkCommand(AmbariSolrCloudClient client, SolrZkClient zkClient, SolrZooKeeper solrZooKeeper) throws Exception {
- List<ACL> acls = solrZooKeeper.getACL(client.getZnode(), new Stat());
- List<String> newUsers = new ArrayList<>();
- Set<String> existingUsers = AclUtils.getUsersFromAclData(acls);
- String saslUsers = client.getSaslUsers();
- if (StringUtils.isNotEmpty(saslUsers)) {
- String[] saslUserNames = saslUsers.split(",");
- for (String saslUser : saslUserNames) {
- if (!existingUsers.contains(saslUser)) {
- acls.add(new ACL(ZooDefs.Perms.ALL, new Id("sasl", saslUser)));
- newUsers.add(saslUser);
- }
- }
- }
- acls = AclUtils.updatePermissionForScheme(acls, "world", ZooDefs.Perms.READ);
- solrZooKeeper.setACL(client.getZnode(), acls, -1);
- return StringUtils.join(newUsers, ",");
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java
index faa21d4..31ad5d3 100644
--- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java
+++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/commands/SecureSolrZNodeZkCommand.java
@@ -23,6 +23,8 @@ import org.apache.ambari.logsearch.solr.util.AclUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.SolrZooKeeper;
+import org.apache.solr.common.cloud.ZkConfigManager;
+import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Id;
@@ -52,9 +54,13 @@ public class SecureSolrZNodeZkCommand extends AbstractZookeeperRetryCommand<Bool
String configsPath = String.format("%s/%s", zNode, "configs");
String collectionsPath = String.format("%s/%s", zNode, "collections");
- List<String> exlustePaths = Arrays.asList(configsPath, collectionsPath);
+ String aliasesPath = String.format("%s/%s", zNode, "aliases.json"); // TODO: protect this later somehow
+ List<String> excludePaths = Arrays.asList(configsPath, collectionsPath, aliasesPath);
- AclUtils.setRecursivelyOn(client.getSolrZkClient().getSolrZooKeeper(), zNode, newAclList, exlustePaths);
+ createZnodeIfNeeded(configsPath, client.getSolrZkClient());
+ createZnodeIfNeeded(collectionsPath, client.getSolrZkClient());
+
+ AclUtils.setRecursivelyOn(client.getSolrZkClient().getSolrZooKeeper(), zNode, newAclList, excludePaths);
List<ACL> commonConfigAcls = new ArrayList<>();
commonConfigAcls.addAll(saslUserList);
@@ -71,4 +77,11 @@ public class SecureSolrZNodeZkCommand extends AbstractZookeeperRetryCommand<Bool
return true;
}
+
+ private void createZnodeIfNeeded(String configsPath, SolrZkClient zkClient) throws KeeperException, InterruptedException {
+ if (!zkClient.exists(configsPath, true)) {
+ LOG.info("'{}' does not exist. Creating it ...", configsPath);
+ zkClient.makePath(configsPath, true);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java
index 8e0132d..fc25c49 100644
--- a/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java
+++ b/ambari-logsearch/ambari-logsearch-solr-client/src/main/java/org/apache/ambari/logsearch/solr/util/AclUtils.java
@@ -34,36 +34,6 @@ import java.util.Set;
public class AclUtils {
- public static Set<String> getUsersFromAclData(List<ACL> acls) {
- Set<String> result = new HashSet<>();
- if (!acls.isEmpty()) {
- for (ACL acl : acls) {
- String username = "";
- String id = acl.getId().getId();
- String[] splitted = id.split(":");
- if (splitted.length > 1) {
- username = splitted[0];
- } else {
- username = id;
- }
- result.add(username);
- }
- }
- return result;
- }
-
- public static List<ACL> updatePermissionForScheme(List<ACL> acls, String scheme, int permission) {
- List<ACL> aclResult = new ArrayList<>();
- if (!acls.isEmpty()) {
- for (ACL acl : acls) {
- int permissionToAdd = scheme.equals(acl.getId().getScheme()) ? permission : acl.getPerms();
- acl.setPerms(permissionToAdd);
- aclResult.add(acl);
- }
- }
- return aclResult;
- }
-
public static List<ACL> mergeAcls(List<ACL> originalAcls, List<ACL> updateAcls) {
Map<String, ACL> aclMap = new HashMap<>();
List<ACL> acls = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/ambari/blob/548a0b6a/ambari-logsearch/docker/bin/start.sh
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/bin/start.sh b/ambari-logsearch/docker/bin/start.sh
index 81c9e66..4c60981 100644
--- a/ambari-logsearch/docker/bin/start.sh
+++ b/ambari-logsearch/docker/bin/start.sh
@@ -81,12 +81,6 @@ function start_solr() {
}
function start_logsearch() {
- echo "Upload configuration sets ..."
-
- $ZKCLI -zkhost localhost:9983 -cmd upconfig -confdir $LOGSEARCH_SERVER_PATH/solr_configsets/audit_logs/conf -confname audit_logs
- $ZKCLI -zkhost localhost:9983 -cmd upconfig -confdir $LOGSEARCH_SERVER_PATH/solr_configsets/hadoop_logs/conf -confname hadoop_logs
- $ZKCLI -zkhost localhost:9983 -cmd upconfig -confdir $LOGSEARCH_SERVER_PATH/solr_configsets/history/conf -confname history
-
$LOGSEARCH_SERVER_PATH/run.sh
touch /var/log/ambari-logsearch-portal/logsearch-app.log
}