You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2016/09/12 11:30:37 UTC
[5/6] ambari git commit: AMBARI-18310. Refactor logsearch portal code
- part 3 (oleewere)
http://git-wip-us.apache.org/repos/asf/ambari/blob/3013589a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
new file mode 100644
index 0000000..ed93b4d
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.dao;
+
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.commons.lang.StringUtils;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+
+@Component
+public class SolrCollectionDao {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SolrCollectionDao.class);
+
+ private static final String ROUTER_FIELD = "_router_field_";
+ private static final int SETUP_RETRY_SECOND = 30;
+
+ /**
+ * This will try to get the collections from the Solr. Ping doesn't work if
+ * collection is not given
+ */
+ public boolean checkSolrStatus(CloudSolrClient cloudSolrClient) {
+ int waitDurationMS = 3 * 60 * 1000;
+ boolean status = false;
+ try {
+ long beginTimeMS = System.currentTimeMillis();
+ long waitIntervalMS = 2000;
+ int pingCount = 0;
+ while (true) {
+ pingCount++;
+ try {
+ List<String> collectionList = getCollections(cloudSolrClient);
+ if (collectionList != null) {
+ LOG.info("checkSolrStatus(): Solr getCollections() is success. collectionList=" + collectionList);
+ status = true;
+ break;
+ }
+ } catch (Exception ex) {
+ LOG.error("Error while doing Solr check", ex);
+ }
+ if (System.currentTimeMillis() - beginTimeMS > waitDurationMS) {
+ LOG.error("Solr is not reachable even after " + (System.currentTimeMillis() - beginTimeMS) + " ms. " +
+ "If you are using alias, then you might have to restart LogSearch after Solr is up and running.");
+ break;
+ } else {
+ LOG.warn("Solr is not not reachable yet. getCollections() attempt count=" + pingCount + ". " +
+ "Will sleep for " + waitIntervalMS + " ms and try again.");
+ }
+ Thread.sleep(waitIntervalMS);
+
+ }
+ } catch (Throwable t) {
+ LOG.error("Seems Solr is not up.");
+ }
+ return status;
+ }
+
+ public void setupCollections(final CloudSolrClient solrClient, final SolrPropsConfig solrPropsConfig) throws Exception {
+ boolean setupStatus = createCollectionsIfNeeded(solrClient, solrPropsConfig);
+ LOG.info("Setup status for " + solrPropsConfig.getCollection() + " is " + setupStatus);
+ if (!setupStatus) {
+ // Start a background thread to do setup
+ Thread setupThread = new Thread("setup_collection_" + solrPropsConfig.getCollection()) {
+ @Override
+ public void run() {
+ LOG.info("Started monitoring thread to check availability of Solr server. collection=" + solrPropsConfig.getCollection());
+ int retryCount = 0;
+ while (true) {
+ try {
+ Thread.sleep(SETUP_RETRY_SECOND * 1000);
+ retryCount++;
+ boolean setupStatus = createCollectionsIfNeeded(solrClient, solrPropsConfig);
+ if (setupStatus) {
+ LOG.info("Setup for collection " + solrPropsConfig.getCollection() + " is successful. Exiting setup retry thread");
+ break;
+ }
+ } catch (InterruptedException sleepInterrupted) {
+ LOG.info("Sleep interrupted while setting up collection " + solrPropsConfig.getCollection());
+ break;
+ } catch (Exception e) {
+ LOG.error("Error setting up collection=" + solrPropsConfig.getCollection(), e);
+ }
+ LOG.error("Error setting collection. collection=" + solrPropsConfig.getCollection() + ", retryCount=" + retryCount);
+ }
+ }
+ };
+ setupThread.setDaemon(true);
+ setupThread.start();
+ }
+ }
+
+ private boolean createCollectionsIfNeeded(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) {
+ boolean result = false;
+ try {
+ List<String> allCollectionList = getCollections(solrClient);
+ if (solrPropsConfig.getSplitInterval().equalsIgnoreCase("none")) {
+ result = createCollection(solrClient, solrPropsConfig, allCollectionList);
+ } else {
+ result = setupCollectionsWithImplicitRouting(solrClient, solrPropsConfig, allCollectionList);
+ }
+ } catch (Exception ex) {
+ LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection(), ex);
+ }
+ return result;
+ }
+
+ @SuppressWarnings("unchecked")
+ public List<String> getCollections(CloudSolrClient solrClient) throws SolrServerException,
+ IOException {
+ try {
+ CollectionAdminRequest.List colListReq = new CollectionAdminRequest.List();
+ CollectionAdminResponse response = colListReq.process(solrClient);
+ if (response.getStatus() != 0) {
+ LOG.error("Error getting collection list from solr. response=" + response);
+ return null;
+ }
+ return (List<String>) response.getResponse().get("collections");
+ } catch (SolrException e) {
+ LOG.error("getCollections() operation failed", e);
+ return new ArrayList<>();
+ }
+ }
+
+ private boolean setupCollectionsWithImplicitRouting(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, List<String> allCollectionList)
+ throws Exception {
+ LOG.info("setupCollectionsWithImplicitRouting(). collectionName=" + solrPropsConfig.getCollection()
+ + ", numberOfShards=" + solrPropsConfig.getNumberOfShards());
+
+ // Default is true, because if the collection and shard is already there, then it will return true
+ boolean returnValue = true;
+
+ List<String> shardsList = new ArrayList<String>();
+ for (int i = 0; i < solrPropsConfig.getNumberOfShards(); i++) {
+ shardsList.add("shard" + i);
+ }
+ String shardsListStr = StringUtils.join(shardsList, ',');
+
+ // Check if collection is already in zookeeper
+ if (!allCollectionList.contains(solrPropsConfig.getCollection())) {
+ LOG.info("Creating collection " + solrPropsConfig.getCollection() + ", shardsList=" + shardsList);
+ CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create();
+ collectionCreateRequest.setCollectionName(solrPropsConfig.getCollection());
+ collectionCreateRequest.setRouterName("implicit");
+ collectionCreateRequest.setShards(shardsListStr);
+ collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards());
+ collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor());
+ collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName());
+ collectionCreateRequest.setRouterField(ROUTER_FIELD);
+ collectionCreateRequest.setMaxShardsPerNode(solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards());
+
+ CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient);
+ if (createResponse.getStatus() != 0) {
+ returnValue = false;
+ LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection()
+ + ", shardsList=" + shardsList +", response=" + createResponse);
+ } else {
+ LOG.info("Created collection " + solrPropsConfig.getCollection() + ", shardsList=" + shardsList);
+ }
+ } else {
+ LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Will check whether it has the required shards");
+ Collection<String> existingShards = getShards(solrClient, solrPropsConfig);
+ for (String shard : shardsList) {
+ if (!existingShards.contains(shard)) {
+ try {
+ LOG.info("Going to add Shard " + shard + " to collection " + solrPropsConfig.getCollection());
+ CollectionAdminRequest.CreateShard createShardRequest = new CollectionAdminRequest.CreateShard();
+ createShardRequest.setCollectionName(solrPropsConfig.getCollection());
+ createShardRequest.setShardName(shard);
+ CollectionAdminResponse response = createShardRequest.process(solrClient);
+ if (response.getStatus() != 0) {
+ LOG.error("Error creating shard " + shard + " in collection " + solrPropsConfig.getCollection() + ", response=" + response);
+ returnValue = false;
+ break;
+ } else {
+ LOG.info("Successfully created shard " + shard + " in collection " + solrPropsConfig.getCollection());
+ }
+ } catch (Throwable t) {
+ LOG.error("Error creating shard " + shard + " in collection " + solrPropsConfig.getCollection(), t);
+ returnValue = false;
+ break;
+ }
+ }
+ }
+ }
+ return returnValue;
+ }
+
+ private Collection<String> getShards(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) {
+ Collection<String> list = new HashSet<>();
+ ZkStateReader reader = solrClient.getZkStateReader();
+ Collection<Slice> slices = reader.getClusterState().getSlices(solrPropsConfig.getCollection());
+ for (Slice slice : slices) {
+ for (Replica replica : slice.getReplicas()) {
+ LOG.info("colName=" + solrPropsConfig.getCollection() + ", slice.name=" + slice.getName() + ", slice.state=" + slice.getState() +
+ ", replica.core=" + replica.getStr("core") + ", replica.state=" + replica.getStr("state"));
+ list.add(slice.getName());
+ }
+ }
+ return list;
+ }
+
+ private boolean createCollection(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, List<String> allCollectionList) throws SolrServerException, IOException {
+
+ if (allCollectionList.contains(solrPropsConfig.getCollection())) {
+ LOG.info("Collection " + solrPropsConfig.getCollection() + " is already there. Won't create it");
+ return true;
+ }
+
+ LOG.info("Creating collection " + solrPropsConfig.getCollection() + ", numberOfShards=" + solrPropsConfig.getNumberOfShards() +
+ ", replicationFactor=" + solrPropsConfig.getReplicationFactor());
+
+ CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create();
+ collectionCreateRequest.setCollectionName(solrPropsConfig.getCollection());
+ collectionCreateRequest.setNumShards(solrPropsConfig.getNumberOfShards());
+ collectionCreateRequest.setReplicationFactor(solrPropsConfig.getReplicationFactor());
+ collectionCreateRequest.setConfigName(solrPropsConfig.getConfigName());
+ collectionCreateRequest.setMaxShardsPerNode(solrPropsConfig.getReplicationFactor() * solrPropsConfig.getNumberOfShards());
+ CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient);
+ if (createResponse.getStatus() != 0) {
+ LOG.error("Error creating collection. collectionName=" + solrPropsConfig.getCollection() + ", response=" + createResponse);
+ return false;
+ } else {
+ LOG.info("Created collection " + solrPropsConfig.getCollection() + ", numberOfShards=" + solrPropsConfig.getNumberOfShards() +
+ ", replicationFactor=" + solrPropsConfig.getReplicationFactor());
+ return true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/3013589a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
index ac7f56f..8381948 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
@@ -22,426 +22,47 @@ package org.apache.ambari.logsearch.dao;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Date;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
+import java.util.Map;
-import org.apache.ambari.logsearch.common.ConfigHelper;
import org.apache.ambari.logsearch.common.LogSearchContext;
import org.apache.ambari.logsearch.common.MessageEnums;
-import org.apache.ambari.logsearch.conf.SolrKerberosConfig;
-import org.apache.ambari.logsearch.conf.SolrUserConfig;
import org.apache.ambari.logsearch.manager.ManagerBase.LogType;
+import org.apache.ambari.logsearch.model.response.BarGraphData;
+import org.apache.ambari.logsearch.model.response.NameValueData;
+import org.apache.ambari.logsearch.util.DateUtil;
import org.apache.ambari.logsearch.util.RESTErrorUtil;
-import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
-import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrRequest.METHOD;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpClientUtil;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.schema.SchemaRequest;
-import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.client.solrj.response.UpdateResponse;
-import org.apache.solr.client.solrj.response.schema.SchemaResponse;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.NamedList;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import javax.inject.Inject;
+import org.apache.solr.common.util.SimpleOrderedMap;
public abstract class SolrDaoBase {
- private static final Logger logger = Logger.getLogger(SolrDaoBase.class);
- private static final Logger logPerformance = Logger.getLogger("org.apache.ambari.logsearch.performance");
-
- public HashMap<String, String> schemaFieldsNameMap = new HashMap<String, String>();
- public HashMap<String, String> schemaFieldTypeMap = new HashMap<String, String>();
-
- private static final String ROUTER_FIELD = "_router_field_";
-
- private static final int SETUP_RETRY_SECOND = 30;
- private static final int SETUP_UPDATE_SECOND = 10*60; //10 min
- private static final int ALIAS_SETUP_RETRY_SECOND = 30*60;
- private LogType logType;
-
- @VisibleForTesting
- protected String collectionName = null;
- @VisibleForTesting
- protected SolrClient solrClient = null;
- @VisibleForTesting
- protected CloudSolrClient solrClouldClient = null;
- @VisibleForTesting
- protected boolean isZkConnectString = false;
-
- private String solrDetail = "";
+ private static final Logger LOG_PERFORMANCE = Logger.getLogger("org.apache.ambari.logsearch.performance");
- private boolean populateFieldsThreadActive = false;
+ public Map<String, String> schemaFieldNameMap = new HashMap<>();
+ public Map<String, String> schemaFieldTypeMap = new HashMap<>();
- @Inject
- private SolrKerberosConfig solrKerberosConfig;
- @Inject
- private SolrUserConfig solrUserConfig;
+ private LogType logType;
protected SolrDaoBase(LogType logType) {
this.logType = logType;
}
- protected SolrClient connectToSolr(String url, String zkConnectString, String collection) throws Exception {
- this.collectionName = collection;
- solrDetail = "zkConnectString=" + zkConnectString + ", collection=" + collection + ", url=" + url;
-
- logger.info("connectToSolr() " + solrDetail);
- if (StringUtils.isBlank(collection)) {
- throw new Exception("For solr, collection name is mandatory. " + solrDetail);
- }
-
- setupSecurity();
-
- if (solrClient != null) {
- return solrClient;
- }
-
- if (!StringUtils.isBlank(zkConnectString)) {
- isZkConnectString = true;
- solrDetail = "zkConnectString=" + zkConnectString + ", collection=" + collection;
- logger.info("Using zookeeper. " + solrDetail);
- solrClouldClient = new CloudSolrClient(zkConnectString);
- solrClouldClient.setDefaultCollection(collection);
- solrClient = solrClouldClient;
- int waitDurationMS = 3 * 60 * 1000;
- checkSolrStatus(waitDurationMS);
- } else {
- if (StringUtils.isBlank(url)) {
- throw new Exception("Both zkConnectString and URL are empty. zkConnectString=" + zkConnectString + ", " +
- "collection=" + collection + ", url=" + url);
- }
- solrDetail = "collection=" + collection + ", url=" + url;
- String collectionURL = url + "/" + collection;
- logger.info("Connecting to solr : " + collectionURL);
- solrClient = new HttpSolrClient(collectionURL);
- }
- return solrClient;
- }
-
- /**
- * This will try to get the collections from the Solr. Ping doesn't work if
- * collection is not given
- */
- protected boolean checkSolrStatus(int waitDurationMS) {
- boolean status = false;
- try {
- long beginTimeMS = System.currentTimeMillis();
- long waitIntervalMS = 2000;
- int pingCount = 0;
- while (true) {
- pingCount++;
- try {
- List<String> collectionList = getCollections();
- if (collectionList != null) {
- logger.info("checkSolrStatus(): Solr getCollections() is success. solr=" + solrDetail + ", collectionList=" + collectionList);
- status = true;
- break;
- }
- } catch (Exception ex) {
- logger.error("Error while doing Solr check", ex);
- }
- if (System.currentTimeMillis() - beginTimeMS > waitDurationMS) {
- logger.error("Solr is not reachable even after " + (System.currentTimeMillis() - beginTimeMS) + " ms. " +
- "If you are using alias, then you might have to restart LogSearch after Solr is up and running. solr=" + solrDetail);
- break;
- } else {
- logger.warn("Solr is not not reachable yet. getCollections() attempt count=" + pingCount + ". " +
- "Will sleep for " + waitIntervalMS + " ms and try again." + " solr=" + solrDetail);
- }
- Thread.sleep(waitIntervalMS);
-
- }
- } catch (Throwable t) {
- logger.error("Seems Solr is not up. solrDetail=" + solrDetail);
- }
- return status;
- }
-
- protected void setupAlias(final String aliasNameIn, final Collection<String> collectionListIn ) throws Exception {
- if (aliasNameIn == null || collectionListIn == null || collectionListIn.size() == 0 || solrClouldClient == null) {
- logger.info("Will not create alias " + aliasNameIn + " for " +
- (collectionListIn == null ? null: collectionListIn.toString()) + ", solrCloudClient=" + solrClouldClient);
- return;
- }
-
- logger.info("setupAlias " + aliasNameIn + " for " + (collectionListIn == null ? null: collectionListIn.toString()));
- // Start a background thread to do setup
- Thread setupThread = new Thread("setup_alias_" + aliasNameIn) {
- @Override
- public void run() {
- logger.info("Started monitoring thread to check availability of Solr server. alias=" + aliasNameIn +
- ", collections=" + collectionListIn.toString());
- int retryCount = 0;
- while (true) {
- try {
- int count = createAlias(aliasNameIn, collectionListIn);
- if (count > 0) {
- solrClouldClient.setDefaultCollection(aliasNameIn);
- if (count == collectionListIn.size()) {
- logger.info("Setup for alias " + aliasNameIn + " is successful. Exiting setup retry thread. " +
- "Collections=" + collectionListIn);
- populateSchemaFields();
- break;
- }
- } else {
- logger.warn("Not able to create alias=" + aliasNameIn + ", retryCount=" + retryCount);
- }
- } catch (Exception e) {
- logger.error("Error setting up alias=" + aliasNameIn, e);
- }
- try {
- Thread.sleep(ALIAS_SETUP_RETRY_SECOND * 1000);
- } catch (InterruptedException sleepInterrupted) {
- logger.info("Sleep interrupted while setting up alias " + aliasNameIn);
- break;
- }
- retryCount++;
- }
- }
- };
- setupThread.setDaemon(true);
- setupThread.start();
- }
-
- private int createAlias(String aliasNameIn, Collection<String> collectionListIn) throws SolrServerException, IOException {
- List<String> collectionToAdd = getCollections();
- collectionToAdd.retainAll(collectionListIn);
-
- String collectionsCSV = null;
- if (!collectionToAdd.isEmpty()) {
- collectionsCSV = StringUtils.join(collectionToAdd, ',');
- CollectionAdminRequest.CreateAlias aliasCreateRequest = new CollectionAdminRequest.CreateAlias();
- aliasCreateRequest.setAliasName(aliasNameIn);
- aliasCreateRequest.setAliasedCollections(collectionsCSV);
- CollectionAdminResponse createResponse = aliasCreateRequest.process(solrClouldClient);
- if (createResponse.getStatus() != 0) {
- logger.error("Error creating alias. alias=" + aliasNameIn + ", collectionList=" + collectionsCSV +
- ", solrDetail=" + solrDetail + ", response=" + createResponse);
- return 0;
- }
- }
- if ( collectionToAdd.size() == collectionListIn.size()) {
- logger.info("Created alias for all collections. alias=" + aliasNameIn + ", collectionsCSV=" + collectionsCSV +
- ", solrDetail=" + solrDetail);
- } else {
- logger.info("Created alias for " + collectionToAdd.size() + " out of " + collectionListIn.size() + " collections. " +
- "alias=" + aliasNameIn + ", collectionsCSV=" + collectionsCSV + ", solrDetail=" + solrDetail);
- }
- return collectionToAdd.size();
- }
-
- protected void setupCollections(final String splitInterval, final String configName, final int numberOfShards,
- final int replicationFactor, boolean needToPopulateSchemaField) throws Exception {
- if (isZkConnectString) {
- boolean setupStatus = createCollectionsIfNeeded(splitInterval, configName, numberOfShards, replicationFactor);
- logger.info("Setup status for " + collectionName + " is " + setupStatus);
- if (!setupStatus) {
- // Start a background thread to do setup
- Thread setupThread = new Thread("setup_collection_" + collectionName) {
- @Override
- public void run() {
- logger.info("Started monitoring thread to check availability of Solr server. collection=" + collectionName);
- int retryCount = 0;
- while (true) {
- try {
- Thread.sleep(SETUP_RETRY_SECOND * 1000);
- retryCount++;
- boolean setupStatus = createCollectionsIfNeeded(splitInterval, configName, numberOfShards, replicationFactor);
- if (setupStatus) {
- logger.info("Setup for collection " + collectionName + " is successful. Exiting setup retry thread");
- break;
- }
- } catch (InterruptedException sleepInterrupted) {
- logger.info("Sleep interrupted while setting up collection " + collectionName);
- break;
- } catch (Exception e) {
- logger.error("Error setting up collection=" + collectionName, e);
- }
- logger.error("Error setting collection. collection=" + collectionName + ", retryCount=" + retryCount);
- }
- }
- };
- setupThread.setDaemon(true);
- setupThread.start();
- }
- }
-
- if (needToPopulateSchemaField){
- populateSchemaFields();
- }
- }
-
- private boolean createCollectionsIfNeeded(String splitInterval, String configName, int numberOfShards, int replicationFactor) {
- boolean result = false;
- try {
- List<String> allCollectionList = getCollections();
- if (splitInterval.equalsIgnoreCase("none")) {
- result = createCollection(configName, numberOfShards, replicationFactor, allCollectionList);
- } else {
- result = setupCollectionsWithImplicitRouting(configName, numberOfShards, replicationFactor, allCollectionList);
- }
- } catch (Exception ex) {
- logger.error("Error creating collection. collectionName=" + collectionName, ex);
- }
- return result;
- }
-
- private List<String> getCollections() throws SolrServerException,
- IOException {
- try {
- CollectionAdminRequest.List colListReq = new CollectionAdminRequest.List();
- CollectionAdminResponse response = colListReq.process(solrClient);
- if (response.getStatus() != 0) {
- logger.error("Error getting collection list from solr. response=" + response);
- return null;
- }
-
- @SuppressWarnings("unchecked")
- List<String> allCollectionList = (List<String>) response.getResponse().get("collections");
- return allCollectionList;
- } catch (SolrException e) {
- logger.error(e);
- return new ArrayList<String>();
- }
- }
-
- private boolean setupCollectionsWithImplicitRouting(String configName, int numberOfShards, int replicationFactor,
- List<String> allCollectionList) throws Exception {
- logger.info("setupCollectionsWithImplicitRouting(). collectionName=" + collectionName + ", numberOfShards=" + numberOfShards);
-
- // Default is true, because if the collection and shard is already there, then it will return true
- boolean returnValue = true;
-
- List<String> shardsList = new ArrayList<String>();
- for (int i = 0; i < numberOfShards; i++) {
- shardsList.add("shard" + i);
- }
- String shardsListStr = StringUtils.join(shardsList, ',');
-
- // Check if collection is already in zookeeper
- if (!allCollectionList.contains(collectionName)) {
- logger.info("Creating collection " + collectionName + ", shardsList=" + shardsList + ", solrDetail=" + solrDetail);
- CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create();
- collectionCreateRequest.setCollectionName(collectionName);
- collectionCreateRequest.setRouterName("implicit");
- collectionCreateRequest.setShards(shardsListStr);
- collectionCreateRequest.setNumShards(numberOfShards);
- collectionCreateRequest.setReplicationFactor(replicationFactor);
- collectionCreateRequest.setConfigName(configName);
- collectionCreateRequest.setRouterField(ROUTER_FIELD);
- collectionCreateRequest.setMaxShardsPerNode(replicationFactor * numberOfShards);
-
- CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient);
- if (createResponse.getStatus() != 0) {
- returnValue = false;
- logger.error("Error creating collection. collectionName=" + collectionName + ", shardsList=" + shardsList +
- ", solrDetail=" + solrDetail + ", response=" + createResponse);
- } else {
- logger.info("Created collection " + collectionName + ", shardsList=" + shardsList + ", solrDetail=" + solrDetail);
- }
- } else {
- logger.info("Collection " + collectionName + " is already there. Will check whether it has the required shards");
- Collection<String> existingShards = getShards();
- for (String shard : shardsList) {
- if (!existingShards.contains(shard)) {
- try {
- logger.info("Going to add Shard " + shard + " to collection " + collectionName);
- CollectionAdminRequest.CreateShard createShardRequest = new CollectionAdminRequest.CreateShard();
- createShardRequest.setCollectionName(collectionName);
- createShardRequest.setShardName(shard);
- CollectionAdminResponse response = createShardRequest.process(solrClient);
- if (response.getStatus() != 0) {
- logger.error("Error creating shard " + shard + " in collection " + collectionName + ", response=" + response +
- ", solrDetail=" + solrDetail);
- returnValue = false;
- break;
- } else {
- logger.info("Successfully created shard " + shard + " in collection " + collectionName);
- }
- } catch (Throwable t) {
- logger.error("Error creating shard " + shard + " in collection " + collectionName + ", solrDetail=" + solrDetail, t);
- returnValue = false;
- break;
- }
- }
- }
- }
- return returnValue;
- }
-
- private Collection<String> getShards() {
- Collection<String> list = new HashSet<String>();
-
- if (solrClouldClient == null) {
- logger.error("getShards(). Only supporting in SolrCloud mode");
- return list;
- }
-
- ZkStateReader reader = solrClouldClient.getZkStateReader();
- Collection<Slice> slices = reader.getClusterState().getSlices(collectionName);
- for (Slice slice : slices) {
- for (Replica replica : slice.getReplicas()) {
- logger.info("colName=" + collectionName + ", slice.name=" + slice.getName() + ", slice.state=" + slice.getState() +
- ", replica.core=" + replica.getStr("core") + ", replica.state=" + replica.getStr("state"));
- list.add(slice.getName());
- }
- }
- return list;
- }
-
- private boolean createCollection(String configName, int numberOfShards, int replicationFactor,
- List<String> allCollectionList) throws SolrServerException, IOException {
- if (allCollectionList.contains(collectionName)) {
- logger.info("Collection " + collectionName + " is already there. Won't create it");
- return true;
- }
-
- logger.info("Creating collection " + collectionName + ", numberOfShards=" + numberOfShards +
- ", replicationFactor=" + replicationFactor + ", solrDetail=" + solrDetail);
-
- CollectionAdminRequest.Create collectionCreateRequest = new CollectionAdminRequest.Create();
- collectionCreateRequest.setCollectionName(collectionName);
- collectionCreateRequest.setNumShards(numberOfShards);
- collectionCreateRequest.setReplicationFactor(replicationFactor);
- collectionCreateRequest.setConfigName(configName);
- collectionCreateRequest.setMaxShardsPerNode(replicationFactor * numberOfShards);
- CollectionAdminResponse createResponse = collectionCreateRequest.process(solrClient);
- if (createResponse.getStatus() != 0) {
- logger.error("Error creating collection. collectionName=" + collectionName + ", solrDetail=" + solrDetail + ", response=" +
- createResponse);
- return false;
- } else {
- logger.info("Created collection " + collectionName + ", numberOfShards=" + numberOfShards +
- ", replicationFactor=" + replicationFactor + ", solrDetail=" + solrDetail);
- return true;
- }
- }
-
public QueryResponse process(SolrQuery solrQuery) throws SolrServerException, IOException {
- if (solrClient != null) {
+ if (getSolrClient() != null) {
String event = solrQuery.get("event");
solrQuery.remove("event");
- QueryResponse queryResponse = solrClient.query(solrQuery, METHOD.POST);
-
+ QueryResponse queryResponse = getSolrClient().query(solrQuery, METHOD.POST);
if (event != null && !"/audit/logs/live/count".equalsIgnoreCase(event)) {
- logPerformance.info("\n Username :- " + LogSearchContext.getCurrentUsername() + " Event :- " + event + " SolrQuery :- " +
+ LOG_PERFORMANCE.info("\n Username :- " + LogSearchContext.getCurrentUsername() + " Event :- " + event + " SolrQuery :- " +
solrQuery + "\nQuery Time Execution :- " + queryResponse.getQTime() + " Total Time Elapsed is :- " +
queryResponse.getElapsedTime());
}
@@ -452,89 +73,34 @@ public abstract class SolrDaoBase {
}
}
- public UpdateResponse addDocs(SolrInputDocument doc) throws SolrServerException, IOException, SolrException {
- UpdateResponse updateResoponse = solrClient.add(doc);
- logPerformance.info("\n Username :- " + LogSearchContext.getCurrentUsername() +
- " Update Time Execution :- " + updateResoponse.getQTime() + " Total Time Elapsed is :- " + updateResoponse.getElapsedTime());
- solrClient.commit();
- return updateResoponse;
- }
+ @SuppressWarnings("unchecked")
+ public void extractValuesFromBuckets(SimpleOrderedMap<Object> jsonFacetResponse, String outerField, String innerField,
+ List<BarGraphData> histogramData) {
+ NamedList<Object> stack = (NamedList<Object>) jsonFacetResponse.get(outerField);
+ ArrayList<Object> stackBuckets = (ArrayList<Object>) stack.get("buckets");
+ for (Object temp : stackBuckets) {
+ BarGraphData vBarGraphData = new BarGraphData();
- public UpdateResponse removeDoc(String query) throws SolrServerException, IOException, SolrException {
- UpdateResponse updateResoponse = solrClient.deleteByQuery(query);
- solrClient.commit();
- logPerformance.info("\n Username :- " + LogSearchContext.getCurrentUsername() +
- " Remove Time Execution :- " + updateResoponse.getQTime() + " Total Time Elapsed is :- " + updateResoponse.getElapsedTime());
- return updateResoponse;
- }
+ SimpleOrderedMap<Object> level = (SimpleOrderedMap<Object>) temp;
+ String name = ((String) level.getVal(0)).toUpperCase();
+ vBarGraphData.setName(name);
- private void setupSecurity() {
- String jaasFile = solrKerberosConfig.getJaasFile();
- boolean securityEnabled = solrKerberosConfig.isEnabled();
- if (securityEnabled) {
- System.setProperty("java.security.auth.login.config", jaasFile);
- HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
- logger.info("setupSecurity() called for kerberos configuration, jaas file: " + jaasFile);
- }
- }
+ Collection<NameValueData> vNameValues = new ArrayList<>();
+ vBarGraphData.setDataCount(vNameValues);
+ ArrayList<Object> levelBuckets = (ArrayList<Object>) ((NamedList<Object>) level.get(innerField)).get("buckets");
+ for (Object temp1 : levelBuckets) {
+ SimpleOrderedMap<Object> countValue = (SimpleOrderedMap<Object>) temp1;
+ String value = DateUtil.convertDateWithMillisecondsToSolrDate((Date) countValue.getVal(0));
- private void populateSchemaFields() {
- if (!populateFieldsThreadActive) {
- populateFieldsThreadActive = true;
- logger.info("Creating thread to populated fields for collection=" + collectionName);
- Thread fieldPopulationThread = new Thread("populated_fields_" + collectionName) {
- @Override
- public void run() {
- logger.info("Started thread to get fields for collection=" + collectionName);
- int retryCount = 0;
- while (true) {
- try {
- Thread.sleep(SETUP_RETRY_SECOND * 1000);
- retryCount++;
- boolean _result = _populateSchemaFields();
- if (_result) {
- logger.info("Populate fields for collection " + collectionName + " is success, Update it after " +
- SETUP_UPDATE_SECOND + " sec");
- Thread.sleep(SETUP_UPDATE_SECOND * 1000);
- }
- } catch (InterruptedException sleepInterrupted) {
- logger.info("Sleep interrupted while populating fields for collection " + collectionName);
- break;
- } catch (Exception ex) {
- logger.error("Error while populating fields for collection " + collectionName + ", retryCount=" + retryCount);
- }
- }
- populateFieldsThreadActive = false;
- logger.info("Exiting thread for populating fields. collection=" + collectionName);
- }
- };
- fieldPopulationThread.setDaemon(true);
- fieldPopulationThread.start();
- }
- }
-
- /**
- * Called from the thread. Don't call this directly
- */
- private boolean _populateSchemaFields() {
- SolrRequest<SchemaResponse> request = new SchemaRequest();
- request.setMethod(METHOD.GET);
- request.setPath("/schema");
- String historyCollection = solrUserConfig.getCollection();
- if (solrClient != null && !collectionName.equals(historyCollection)) {
- NamedList<Object> namedList = null;
- try {
- namedList = solrClient.request(request);
- logger.debug("populateSchemaFields() collection=" + collectionName + ", fields=" + namedList);
- } catch (SolrException | SolrServerException | IOException e) {
- logger.error("Error occured while popuplating field. collection=" + collectionName, e);
- }
-
- if (namedList != null) {
- ConfigHelper.extractSchemaFieldsName(namedList.toString(), schemaFieldsNameMap,schemaFieldTypeMap);
- return true;
+ String count = "" + countValue.getVal(1);
+ NameValueData vNameValue = new NameValueData();
+ vNameValue.setName(value);
+ vNameValue.setValue(count);
+ vNameValues.add(vNameValue);
}
+ histogramData.add(vBarGraphData);
}
- return false;
}
+
+ public abstract CloudSolrClient getSolrClient();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/3013589a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
new file mode 100644
index 0000000..b6764d0
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.dao;
+
+import org.apache.ambari.logsearch.common.LogSearchConstants;
+import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.conf.SolrPropsConfig;
+import org.apache.ambari.logsearch.conf.SolrUserPropsConfig;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.schema.SchemaRequest;
+import org.apache.solr.client.solrj.response.schema.SchemaResponse;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.NamedList;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+@Component
+@Scope(value = "prototype")
+public class SolrSchemaFieldDao {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SolrSchemaFieldDao.class);
+
+ private static final int SETUP_RETRY_SECOND = 30;
+ private static final int SETUP_UPDATE_SECOND = 10 * 60; // 10 min
+
+ private boolean populateFieldsThreadActive = false;
+
+ @Inject
+ private SolrUserPropsConfig solrUserPropsConfig;
+
+ public void populateSchemaFields(final CloudSolrClient solrClient, final SolrPropsConfig solrPropsConfig,
+ final SolrDaoBase solrDao) {
+ if (!populateFieldsThreadActive) {
+ populateFieldsThreadActive = true;
+ LOG.info("Creating thread to populated fields for collection=" + solrPropsConfig.getCollection());
+ Thread fieldPopulationThread = new Thread("populated_fields_" + solrPropsConfig.getCollection()) {
+ @Override
+ public void run() {
+ LOG.info("Started thread to get fields for collection=" + solrPropsConfig.getCollection());
+ int retryCount = 0;
+ while (true) {
+ try {
+ Thread.sleep(SETUP_RETRY_SECOND * 1000);
+ retryCount++;
+ boolean _result = _populateSchemaFields(solrClient, solrPropsConfig, solrDao);
+ if (_result) {
+ LOG.info("Populate fields for collection " + solrPropsConfig.getCollection() + " is success, Update it after " +
+ SETUP_UPDATE_SECOND + " sec");
+ Thread.sleep(SETUP_UPDATE_SECOND * 1000);
+ }
+ } catch (InterruptedException sleepInterrupted) {
+ LOG.info("Sleep interrupted while populating fields for collection " + solrPropsConfig.getCollection());
+ break;
+ } catch (Exception ex) {
+ LOG.error("Error while populating fields for collection " + solrPropsConfig.getCollection() + ", retryCount=" + retryCount);
+ }
+ }
+ populateFieldsThreadActive = false;
+ LOG.info("Exiting thread for populating fields. collection=" + solrPropsConfig.getCollection());
+ }
+ };
+ fieldPopulationThread.setDaemon(true);
+ fieldPopulationThread.start();
+ }
+ }
+
+ /**
+ * Called from the thread. Don't call this directly
+ */
+ private boolean _populateSchemaFields(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, SolrDaoBase solrDao) {
+ SolrRequest<SchemaResponse> request = new SchemaRequest();
+ request.setMethod(SolrRequest.METHOD.GET);
+ request.setPath("/schema");
+ String historyCollection = solrUserPropsConfig.getCollection();
+ if (solrClient != null && !solrPropsConfig.getCollection().equals(historyCollection)) {
+ NamedList<Object> namedList = null;
+ try {
+ namedList = solrClient.request(request);
+ LOG.debug("populateSchemaFields() collection=" + solrPropsConfig.getCollection() + ", fields=" + namedList);
+ } catch (SolrException | SolrServerException | IOException e) {
+ LOG.error("Error occured while popuplating field. collection=" + solrPropsConfig.getCollection(), e);
+ }
+
+ if (namedList != null) {
+ extractSchemaFieldsName(namedList.toString(), solrDao.schemaFieldNameMap, solrDao.schemaFieldTypeMap);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public void extractSchemaFieldsName(String responseString,
+ final Map<String, String> schemaFieldsNameMap,
+ final Map<String, String> schemaFieldTypeMap) {
+ try {
+ JSONObject jsonObject = new JSONObject(responseString);
+ JSONObject schemajsonObject = jsonObject.getJSONObject("schema");
+ JSONArray jsonArrayList = schemajsonObject.getJSONArray("fields");
+ JSONArray fieldTypeJsonArray = schemajsonObject
+ .getJSONArray("fieldTypes");
+ if (jsonArrayList == null) {
+ return;
+ }
+ if (fieldTypeJsonArray == null) {
+ return;
+ }
+ HashMap<String, String> _schemaFieldTypeMap = new HashMap<>();
+ HashMap<String, String> _schemaFieldsNameMap = new HashMap<String, String>();
+ for (int i = 0; i < fieldTypeJsonArray.length(); i++) {
+ JSONObject typeObject = fieldTypeJsonArray.getJSONObject(i);
+ String name = typeObject.getString("name");
+ String fieldTypeJson = typeObject.toString();
+ _schemaFieldTypeMap.put(name, fieldTypeJson);
+ }
+ for (int i = 0; i < jsonArrayList.length(); i++) {
+ JSONObject explrObject = jsonArrayList.getJSONObject(i);
+ String name = explrObject.getString("name");
+ String type = explrObject.getString("type");
+ if (!name.contains("@") && !name.startsWith("_") && !name.contains("_md5") && !name.contains("_ms") &&
+ !name.contains(LogSearchConstants.NGRAM_SUFFIX) && !name.contains("tags") && !name.contains("_str")) {
+ _schemaFieldsNameMap.put(name, type);
+ }
+ }
+ schemaFieldsNameMap.clear();
+ schemaFieldTypeMap.clear();
+ schemaFieldsNameMap.putAll(_schemaFieldsNameMap);
+ schemaFieldTypeMap.putAll(_schemaFieldTypeMap);
+ } catch (Exception e) {
+ LOG.error(e + "Credentials not specified in logsearch.properties " + MessageEnums.ERROR_SYSTEM);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/3013589a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
index a9fb8d2..88c4577 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java
@@ -29,12 +29,15 @@ import java.util.Scanner;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
-import org.apache.ambari.logsearch.conf.SolrUserConfig;
+import org.apache.ambari.logsearch.common.LogSearchContext;
+import org.apache.ambari.logsearch.conf.SolrUserPropsConfig;
import org.apache.ambari.logsearch.view.VLogfeederFilterWrapper;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
@@ -47,38 +50,55 @@ import com.google.gson.JsonParseException;
import org.apache.ambari.logsearch.manager.ManagerBase.LogType;
import org.apache.ambari.logsearch.util.JSONUtil;
import org.apache.log4j.Logger;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.solr.core.SolrTemplate;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component
public class UserConfigSolrDao extends SolrDaoBase {
- private static final Logger logger = Logger.getLogger(UserConfigSolrDao.class);
+ private static final Logger LOG = Logger.getLogger(UserConfigSolrDao.class);
+
+ private static final Logger LOG_PERFORMANCE = Logger.getLogger("org.apache.ambari.logsearch.performance");
+
+ @Inject
+ private SolrUserPropsConfig solrUserConfig;
+
+ @Inject
+ private SolrCollectionDao solrCollectionDao;
+
+ @Inject
+ private SolrSchemaFieldDao solrSchemaFieldDao;
@Inject
- private SolrUserConfig solrUserConfig;
+ @Qualifier("userConfigSolrTemplate")
+ private SolrTemplate userConfigSolrTemplate;
public UserConfigSolrDao() {
super(LogType.SERVICE);
}
+ @Override
+ public CloudSolrClient getSolrClient() {
+ return (CloudSolrClient) userConfigSolrTemplate.getSolrClient();
+ }
+
+
@PostConstruct
public void postConstructor() {
String solrUrl = solrUserConfig.getSolrUrl();
String zkConnectString = solrUserConfig.getZkConnectString();
String collection = solrUserConfig.getCollection();
- String configName = solrUserConfig.getConfigName();
- int replicationFactor = solrUserConfig.getReplicationFactor();
- String splitInterval = solrUserConfig.getSplitInterval();
- int numberOfShards = solrUserConfig.getNumberOfShards();
try {
- connectToSolr(solrUrl, zkConnectString, collection);
- setupCollections(splitInterval, configName, numberOfShards, replicationFactor, true);
+ solrCollectionDao.checkSolrStatus(getSolrClient());
+ solrCollectionDao.setupCollections(getSolrClient(), solrUserConfig);
+ solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrUserConfig, this);
intializeLogFeederFilter();
} catch (Exception e) {
- logger.error("error while connecting to Solr for history logs : solrUrl=" + solrUrl + ", zkConnectString=" + zkConnectString +
+ LOG.error("error while connecting to Solr for history logs : solrUrl=" + solrUrl + ", zkConnectString=" + zkConnectString +
", collection=" + collection, e);
}
}
@@ -87,7 +107,7 @@ public class UserConfigSolrDao extends SolrDaoBase {
try {
getUserFilter();
} catch (SolrServerException | IOException e) {
- logger.error("not able to save logfeeder filter while initialization", e);
+ LOG.error("not able to save logfeeder filter while initialization", e);
}
}
@@ -107,6 +127,22 @@ public class UserConfigSolrDao extends SolrDaoBase {
removeDoc("id:" + id);
}
+ public UpdateResponse addDocs(SolrInputDocument doc) throws SolrServerException, IOException, SolrException {
+ UpdateResponse updateResoponse = getSolrClient().add(doc);
+ LOG_PERFORMANCE.info("\n Username :- " + LogSearchContext.getCurrentUsername() +
+ " Update Time Execution :- " + updateResoponse.getQTime() + " Total Time Elapsed is :- " + updateResoponse.getElapsedTime());
+ getSolrClient().commit();
+ return updateResoponse;
+ }
+
+ public UpdateResponse removeDoc(String query) throws SolrServerException, IOException, SolrException {
+ UpdateResponse updateResoponse = getSolrClient().deleteByQuery(query);
+ getSolrClient().commit();
+ LOG_PERFORMANCE.info("\n Username :- " + LogSearchContext.getCurrentUsername() +
+ " Remove Time Execution :- " + updateResoponse.getQTime() + " Total Time Elapsed is :- " + updateResoponse.getElapsedTime());
+ return updateResoponse;
+ }
+
@SuppressWarnings("unchecked")
public VLogfeederFilterWrapper getUserFilter() throws SolrServerException, IOException {
@@ -158,7 +194,7 @@ public class UserConfigSolrDao extends SolrDaoBase {
saveUserFilter(logfeederFilterWrapper);
} catch (JsonParseException | JSONException je) {
- logger.error("Error parsing JSON. key=" + key + ", componentArray=" + componentArray, je);
+ LOG.error("Error parsing JSON. key=" + key + ", componentArray=" + componentArray, je);
logfeederFilterWrapper = new VLogfeederFilterWrapper();
}
}
@@ -182,7 +218,7 @@ public class UserConfigSolrDao extends SolrDaoBase {
scanner.close();
} catch (IOException e) {
- logger.error("Unable to read HadoopServiceConfig.json", e);
+ LOG.error("Unable to read HadoopServiceConfig.json", e);
}
String hadoopServiceConfig = result.toString();
@@ -191,5 +227,4 @@ public class UserConfigSolrDao extends SolrDaoBase {
}
return null;
}
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/3013589a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java
index 4ca9df6..0810571 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java
@@ -26,17 +26,15 @@ import java.util.HashMap;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
-import org.apache.ambari.logsearch.conf.AuthConfig;
+import org.apache.ambari.logsearch.conf.AuthPropsConfig;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.stereotype.Repository;
-import org.apache.ambari.logsearch.common.PropertiesHelper;
import org.apache.ambari.logsearch.util.CommonUtil;
import org.apache.ambari.logsearch.util.FileUtil;
import org.apache.ambari.logsearch.util.JSONUtil;
import org.apache.ambari.logsearch.web.model.Privilege;
import org.apache.ambari.logsearch.web.model.Role;
import org.apache.ambari.logsearch.web.model.User;
-import org.apache.ambari.logsearch.web.security.LogsearchFileAuthenticationProvider;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate;
import org.apache.commons.lang.StringUtils;
@@ -52,16 +50,16 @@ public class UserDao {
private static final String NAME = "name";
@Inject
- private AuthConfig authConfig;
+ private AuthPropsConfig authPropsConfig;
private ArrayList<HashMap<String, String>> userList = null;
@SuppressWarnings("unchecked")
@PostConstruct
public void initialization() {
- if (authConfig.isAuthFileEnabled()) {
+ if (authPropsConfig.isAuthFileEnabled()) {
try {
- String userPassJsonFileName = authConfig.getCredentialsFile();
+ String userPassJsonFileName = authPropsConfig.getCredentialsFile();
logger.info("USER PASS JSON file NAME:" + userPassJsonFileName);
File jsonFile = FileUtil.getFileFromClasspath(userPassJsonFileName);
if (jsonFile == null || !jsonFile.exists()) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/3013589a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
index 8e14452..0df2ca3 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
@@ -39,9 +39,7 @@ public class DocConstants {
public static final String PAGE_SIZE_D = "Page size of the results";
public static final String UNIT_D = "Aggregate the data with time gap as unit i.e 1MINUTE";
public static final String QUERY_D = "not required";
- public static final String COLUMN_QUERY_D = "not required";
public static final String I_MESSAGE_D = "Include query which will query against message column";
- public static final String G_E_MESSAGE_D = "not required";
public static final String E_MESSAGE_D = "Exclude query which will query against message column";
public static final String IS_LAST_PAGE_D = "Show last page (true/false)";
public static final String FIELD_D = "Get values for particular field";
@@ -64,17 +62,13 @@ public class DocConstants {
public class ServiceDescriptions {
public static final String LEVEL_D = "filter for log level";
- public static final String ADVANCED_SEARCH_D = "not required";
public static final String BUNDLE_ID = "filter for host";
- public static final String TREE_PARAMS_D = "Host hierarchy shown on UI, filtering there is supported by this param";
public static final String FILE_NAME_D = "File name filter which is supported from browser url";
- public static final String HOST_NAME_D = "Host name filter which is supported from browser url";
public static final String DATE_RANGE_LABEL_D = "Date range label (e.g.: Today)";
+ public static final String HOST_NAME_D = "Host name filter which is supported from browser url";
public static final String COMPONENT_NAME_D = "Component name filter which is supported from browser url";
public static final String FIND_D = "Finding particular text on subsequent pages in case of table view with pagination";
public static final String ID_D = "Log id value for traversing to that particular record with that log id";
- public static final String HOST_D = "filter for host";
- public static final String COMPONENT_D = "filter for component";
public static final String KEYWORD_TYPE_D = "Serching the find param value in previous or next in paginated table";
public static final String TOKEN_D = "unique number used along with FIND_D. The request can be canceled using this token";
public static final String SOURCE_LOG_ID_D = "fetch the record set having that log Id";
@@ -106,19 +100,6 @@ public class DocConstants {
public static final String GET_HADOOP_SERVICE_CONFIG_JSON_OD = "Get the json having meta data of services supported by logsearch";
}
-
- public class LogFileDescriptions {
- public static final String HOST_D = "not required";
- public static final String COMPONENT_D = "not required";
- public static final String LOG_TYPE_D = "not required";
- public static final String TAIL_SIZE_D = "not required";
- }
-
- public class LogFileOperationDescriptions {
- public static final String SEARCH_LOG_FILES_OD = "not required";
- public static final String GET_LOG_FILE_TAIL_OD = "not required";
- }
-
public class PublicOperationDescriptions {
public static final String OBTAIN_GENERAL_CONFIG_OD = "Obtain general config";
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/3013589a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
index bc377e5..287ff6c 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java
@@ -63,7 +63,7 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
String from = (String) searchCriteria.getParamValue("from");
String to = (String) searchCriteria.getParamValue("to");
String unit = (String) searchCriteria.getParamValue("unit");
- String typeXAxis = solrDaoBase.schemaFieldsNameMap.get(xAxisField);
+ String typeXAxis = solrDaoBase.schemaFieldNameMap.get(xAxisField);
typeXAxis = (StringUtils.isBlank(typeXAxis)) ? "string" : typeXAxis;
// Y axis credentials
http://git-wip-us.apache.org/repos/asf/ambari/blob/3013589a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
index c64cf71..52482c2 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
@@ -28,17 +28,14 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
-import java.util.HashMap;
import java.util.List;
-import org.apache.ambari.logsearch.common.ConfigHelper;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.common.ManageStartEndTime;
import org.apache.ambari.logsearch.common.MessageEnums;
-import org.apache.ambari.logsearch.conf.SolrAuditLogConfig;
+import org.apache.ambari.logsearch.conf.SolrAuditLogPropsConfig;
import org.apache.ambari.logsearch.dao.AuditSolrDao;
import org.apache.ambari.logsearch.graph.GraphDataGenerator;
import org.apache.ambari.logsearch.model.response.AuditLogResponse;
@@ -63,7 +60,6 @@ import org.apache.ambari.logsearch.util.RESTErrorUtil;
import org.apache.ambari.logsearch.util.SolrUtil;
import org.apache.ambari.logsearch.view.VResponse;
import org.apache.ambari.logsearch.query.model.AuditLogSearchCriteria;
-import org.apache.ambari.logsearch.query.model.SearchCriteria;
import org.apache.log4j.Logger;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
@@ -85,7 +81,7 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
@Inject
private GraphDataGenerator graphDataGenerator;
@Inject
- private SolrAuditLogConfig solrAuditLogConfig;
+ private SolrAuditLogPropsConfig solrAuditLogPropsConfig;
public AuditLogResponse getLogs(AuditLogSearchCriteria searchCriteria) {
Boolean isLastPage = searchCriteria.isLastPage();
@@ -159,29 +155,7 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
String jsonHistogramQuery = queryGenerator.buildJSONFacetTermTimeRangeQuery(LogSearchConstants.AUDIT_COMPONENT,
LogSearchConstants.AUDIT_EVTTIME, from, to, unit).replace("\\", "");
- try {
- SolrUtil.setJSONFacet(solrQuery, jsonHistogramQuery);
- SolrUtil.setRowCount(solrQuery, 0);
- QueryResponse response = auditSolrDao.process(solrQuery);
- if (response == null) {
- return dataList;
- }
- SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response.getResponse().get("facets");
-
- if (jsonFacetResponse == null || jsonFacetResponse.toString().equals("{count=0}")) {
- return dataList;
- }
-
- extractValuesFromBucket(jsonFacetResponse, "x", "y", histogramData);
-
- dataList.setGraphData(histogramData);
- return dataList;
-
- } catch (SolrServerException | SolrException | IOException e) {
- logger.error(e);
- throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM);
-
- }
+ return getGraphResponse(dataList, solrQuery, histogramData, jsonHistogramQuery);
}
@SuppressWarnings({"unchecked", "rawtypes"})
@@ -316,49 +290,11 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
String jsonHistogramQuery = queryGenerator.buildJSONFacetTermTimeRangeQuery(LogSearchConstants.AUDIT_REQUEST_USER,
LogSearchConstants.AUDIT_EVTTIME, from, to, unit).replace("\\", "");
- try {
- SolrUtil.setJSONFacet(solrQuery, jsonHistogramQuery);
- SolrUtil.setRowCount(solrQuery, 0);
- QueryResponse response = auditSolrDao.process(solrQuery);
- if (response == null) {
- return dataList;
- }
- SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response.getResponse().get("facets");
-
- if (jsonFacetResponse == null || jsonFacetResponse.toString().equals("{count=0}")) {
- return dataList;
- }
- extractValuesFromBucket(jsonFacetResponse, "x", "y", histogramData);
-
- dataList.setGraphData(histogramData);
- return dataList;
-
- } catch (SolrException | IOException | SolrServerException e) {
- logger.error("Error during solrQuery=" + solrQuery, e);
- throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM);
- }
-
+ return getGraphResponse(dataList, solrQuery, histogramData, jsonHistogramQuery);
}
public String getAuditLogsSchemaFieldsName() {
- String excludeArray[] = Arrays.copyOf(solrAuditLogConfig.getExcludeColumnList().toArray(),
- solrAuditLogConfig.getExcludeColumnList().size(), String[].class);
- List<String> fieldNames = new ArrayList<String>();
- HashMap<String, String> uiFieldColumnMapping = new HashMap<String, String>();
- ConfigHelper.getSchemaFieldsName(excludeArray, fieldNames, auditSolrDao);
-
- for (String fieldName : fieldNames) {
- String uiField = solrAuditLogConfig.getSolrAndUiColumns().get(fieldName + LogSearchConstants.SOLR_SUFFIX);
- if (uiField == null) {
- uiFieldColumnMapping.put(fieldName, fieldName);
- } else {
- uiFieldColumnMapping.put(fieldName, uiField);
- }
- }
-
- uiFieldColumnMapping = BizUtil.sortHashMapByValues(uiFieldColumnMapping);
- return convertObjToString(uiFieldColumnMapping);
-
+ return convertObjToString(auditSolrDao.schemaFieldNameMap);
}
public BarGraphDataListResponse getAnyGraphData(AnyGraphSearchCriteria searchCriteria) {
@@ -372,35 +308,6 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
}
- @SuppressWarnings("unchecked")
- private void extractValuesFromBucket(SimpleOrderedMap<Object> jsonFacetResponse, String outerField, String innerField,
- List<BarGraphData> histogramData) {
- NamedList<Object> stack = (NamedList<Object>) jsonFacetResponse.get(outerField);
- ArrayList<Object> stackBuckets = (ArrayList<Object>) stack.get("buckets");
- for (Object temp : stackBuckets) {
- BarGraphData vBarGraphData = new BarGraphData();
-
- SimpleOrderedMap<Object> level = (SimpleOrderedMap<Object>) temp;
- String name = ((String) level.getVal(0)).toUpperCase();
- vBarGraphData.setName(name);
-
- Collection<NameValueData> vNameValues = new ArrayList<NameValueData>();
- vBarGraphData.setDataCount(vNameValues);
- ArrayList<Object> levelBuckets = (ArrayList<Object>) ((NamedList<Object>) level.get(innerField)).get("buckets");
- for (Object temp1 : levelBuckets) {
- SimpleOrderedMap<Object> countValue = (SimpleOrderedMap<Object>) temp1;
- String value = DateUtil.convertDateWithMillisecondsToSolrDate((Date) countValue.getVal(0));
-
- String count = "" + countValue.getVal(1);
- NameValueData vNameValue = new NameValueData();
- vNameValue.setName(value);
- vNameValue.setValue(count);
- vNameValues.add(vNameValue);
- }
- histogramData.add(vBarGraphData);
- }
- }
-
@SuppressWarnings({"unchecked"})
public Response exportUserTableToTextFile(UserExportSearchCriteria searchCriteria) {
String jsonUserQuery =
@@ -417,6 +324,7 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
SolrUtil.setRowCount(solrQuery, 0);
String dataFormat = searchCriteria.getFormat();
+
FileOutputStream fis = null;
try {
QueryResponse queryResponse = auditSolrDao.process(solrQuery);
@@ -458,15 +366,7 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
data += addBlank(userName);
Collection<NameValueData> vnameValueList = graphData.getDataCount();
- int count = 0;
- String blank = "";
- for (NameValueData vNameValue : vnameValueList) {
- data += blank + vNameValue.getName() + " " + vNameValue.getValue() + "\n";
- if (count == 0)
- blank = addBlank(blank);
- count++;
-
- }
+ data = appendNameValueData(data, vnameValueList);
while (largeUserName.length() > 0) {
data += largeUserName.substring(0, 45) + "\n";
}
@@ -490,14 +390,7 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
//resourceName = resourceName.replaceAll("(.{45})", resourceName.substring(0, 45)+"\n");
data += addBlank(resourceName);
Collection<NameValueData> vnameValueList = graphData.getDataCount();
- int count = 0;
- String blank = "";
- for (NameValueData vNameValue : vnameValueList) {
- data += blank + vNameValue.getName() + " " + vNameValue.getValue() + "\n";
- if (count == 0)
- blank = addBlank(blank);
- count++;
- }
+ data = appendNameValueData(data, vnameValueList);
String tempLargeResourceName = largeResourceName;
while (largeResourceName.length() > 45) {
largeResourceName = tempLargeResourceName.substring(0, 45);
@@ -539,6 +432,45 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
}
}
+ private BarGraphDataListResponse getGraphResponse(BarGraphDataListResponse dataList, SolrQuery solrQuery,
+ List<BarGraphData> histogramData, String jsonHistogramQuery) {
+ try {
+ SolrUtil.setJSONFacet(solrQuery, jsonHistogramQuery);
+ SolrUtil.setRowCount(solrQuery, 0);
+ QueryResponse response = auditSolrDao.process(solrQuery);
+ if (response == null) {
+ return dataList;
+ }
+ SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response.getResponse().get("facets");
+
+ if (jsonFacetResponse == null || jsonFacetResponse.toString().equals("{count=0}")) {
+ return dataList;
+ }
+
+ auditSolrDao.extractValuesFromBuckets(jsonFacetResponse, "x", "y", histogramData);
+
+ dataList.setGraphData(histogramData);
+ return dataList;
+
+ } catch (SolrServerException | SolrException | IOException e) {
+ logger.error(e);
+ throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM);
+
+ }
+ }
+
+ private String appendNameValueData(String data, Collection<NameValueData> vnameValueList) {
+ int count = 0;
+ String blank = "";
+ for (NameValueData vNameValue : vnameValueList) {
+ data += blank + vNameValue.getName() + " " + vNameValue.getValue() + "\n";
+ if (count == 0)
+ blank = addBlank(blank);
+ count++;
+ }
+ return data;
+ }
+
private String addBlank(String field) {
int blanks = 50;
int strSize = field.length();
http://git-wip-us.apache.org/repos/asf/ambari/blob/3013589a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogFileManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogFileManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogFileManager.java
deleted file mode 100644
index 405eaef..0000000
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogFileManager.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logsearch.manager;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.ambari.logsearch.common.LogSearchConstants;
-import org.apache.ambari.logsearch.common.MessageEnums;
-import org.apache.ambari.logsearch.model.response.LogFileData;
-import org.apache.ambari.logsearch.model.response.LogFileDataListResponse;
-import org.apache.ambari.logsearch.model.response.LogListResponse;
-import org.apache.ambari.logsearch.model.response.ServiceLogData;
-import org.apache.ambari.logsearch.model.response.ServiceLogResponse;
-import org.apache.ambari.logsearch.query.model.SearchCriteria;
-import org.apache.ambari.logsearch.dao.AuditSolrDao;
-import org.apache.ambari.logsearch.dao.ServiceLogsSolrDao;
-import org.apache.ambari.logsearch.dao.SolrDaoBase;
-import org.apache.ambari.logsearch.util.RESTErrorUtil;
-import org.apache.ambari.logsearch.util.SolrUtil;
-import org.apache.commons.io.FilenameUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.log4j.Logger;
-import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.response.FacetField;
-import org.apache.solr.client.solrj.response.FacetField.Count;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.common.SolrException;
-import org.springframework.stereotype.Component;
-
-import javax.inject.Inject;
-
-
-@Component
-public class LogFileManager extends ManagerBase<ServiceLogData, ServiceLogResponse> {
-
- private static final Logger logger = Logger.getLogger(LogFileManager.class);
-
- @Inject
- private ServiceLogsSolrDao serviceLogsSolrDao;
- @Inject
- private AuditSolrDao auditSolrDao;
-
- public LogFileDataListResponse searchLogFiles(SearchCriteria searchCriteria) {
- LogFileDataListResponse logFileList = new LogFileDataListResponse();
- List<LogFileData> logFiles = new ArrayList<LogFileData>();
- String componentName = (String) searchCriteria.getParamValue("component");
- String host = (String) searchCriteria.getParamValue("host");
- int minCount = 1;// to remove zero count facet
- SolrQuery solrQuery = new SolrQuery();
- SolrUtil.setMainQuery(solrQuery, null);
- SolrUtil.setFacetFieldWithMincount(solrQuery, LogSearchConstants.SOLR_PATH, minCount);
- // adding filter
- queryGenerator.setSingleIncludeFilter(solrQuery, LogSearchConstants.SOLR_COMPONENT, componentName);
- queryGenerator.setSingleIncludeFilter(solrQuery, LogSearchConstants.SOLR_HOST, host);
- try {
- String logType = (String) searchCriteria.getParamValue("logType");
- if (StringUtils.isBlank(logType)) {
- logType = LogType.SERVICE.name();// default is service Log
- }
- SolrDaoBase daoMgr = null;
- if (logType.equalsIgnoreCase(LogType.SERVICE.name())) {
- daoMgr = serviceLogsSolrDao;
- } else if (logType.equalsIgnoreCase(LogType.AUDIT.name())) {
- daoMgr = auditSolrDao;
- } else {
- throw RESTErrorUtil.createRESTException(logType + " is not a valid logType", MessageEnums.INVALID_INPUT_DATA);
- }
- QueryResponse queryResponse = daoMgr.process(solrQuery);
- if (queryResponse.getFacetField(LogSearchConstants.SOLR_PATH) != null) {
- FacetField queryFacetField = queryResponse.getFacetField(LogSearchConstants.SOLR_PATH);
- if (queryFacetField != null) {
- List<Count> countList = queryFacetField.getValues();
- for (Count count : countList) {
- LogFileData vLogFile = new LogFileData();
- String filePath = count.getName();
- String fileName = FilenameUtils.getName(filePath);
- vLogFile.setPath(filePath);
- vLogFile.setName(fileName);
- logFiles.add(vLogFile);
- }
- }
- }
- } catch (SolrException | SolrServerException | IOException e) {
- logger.error("Error in solr query :" + e.getLocalizedMessage() + "\n Query :" + solrQuery.toQueryString(), e.getCause());
- throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM);
- }
- logFileList.setLogFiles(logFiles);
-
- return logFileList;
- }
-
- public LogListResponse getLogFileTail(SearchCriteria searchCriteria) {
- String host = (String) searchCriteria.getParamValue("host");
- String logFile = (String) searchCriteria.getParamValue("name");
- String component = (String) searchCriteria.getParamValue("component");
- String tailSize = (String) searchCriteria.getParamValue("tailSize");
- if (StringUtils.isBlank(host)) {
- throw RESTErrorUtil.createRESTException("missing Host Name", MessageEnums.ERROR_SYSTEM);
- }
- tailSize = (StringUtils.isBlank(tailSize)) ? "10" : tailSize;
- SolrQuery logFileTailQuery = new SolrQuery();
- try {
- int tail = Integer.parseInt(tailSize);
- tail = tail > 100 ? 100 : tail;
- SolrUtil.setMainQuery(logFileTailQuery, null);
- queryGenerator.setSingleIncludeFilter(logFileTailQuery, LogSearchConstants.SOLR_HOST, host);
- if (!StringUtils.isBlank(logFile)) {
- queryGenerator.setSingleIncludeFilter(logFileTailQuery, LogSearchConstants.SOLR_PATH, SolrUtil.makeSolrSearchString(logFile));
- } else if (!StringUtils.isBlank(component)) {
- queryGenerator.setSingleIncludeFilter(logFileTailQuery, LogSearchConstants.SOLR_COMPONENT, component);
- } else {
- throw RESTErrorUtil.createRESTException("component or logfile parameter must be present", MessageEnums.ERROR_SYSTEM);
- }
-
- SolrUtil.setRowCount(logFileTailQuery, tail);
- queryGenerator.setSortOrderDefaultServiceLog(logFileTailQuery, new SearchCriteria());
- return getLogAsPaginationProvided(logFileTailQuery, serviceLogsSolrDao);
-
- } catch (NumberFormatException ne) {
-
- throw RESTErrorUtil.createRESTException(ne.getMessage(),
- MessageEnums.ERROR_SYSTEM);
-
- }
- }
-
- @Override
- protected List<ServiceLogData> convertToSolrBeans(QueryResponse response) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- protected ServiceLogResponse createLogSearchResponse() {
- throw new UnsupportedOperationException();
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/3013589a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java
index 3ac2be6..82817d6 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java
@@ -22,7 +22,7 @@ package org.apache.ambari.logsearch.manager;
import java.util.ArrayList;
import java.util.List;
-import org.apache.ambari.logsearch.conf.AuthConfig;
+import org.apache.ambari.logsearch.conf.AuthPropsConfig;
import org.apache.ambari.logsearch.model.response.NameValueData;
import org.apache.ambari.logsearch.model.response.NameValueDataListResponse;
import org.springframework.stereotype.Component;
@@ -33,14 +33,14 @@ import javax.inject.Inject;
public class PublicManager extends JsonManagerBase {
@Inject
- private AuthConfig authConfig;
+ private AuthPropsConfig authPropsConfig;
public String getGeneralConfig() {
NameValueDataListResponse nameValueList = new NameValueDataListResponse();
List<NameValueData> nameValues = new ArrayList<>();
NameValueData nameValue = new NameValueData();
nameValue.setName("simpleAuth");
- nameValue.setValue("" + authConfig.isAuthSimpleEnabled());
+ nameValue.setValue("" + authPropsConfig.isAuthSimpleEnabled());
nameValues.add(nameValue);
nameValueList.setvNameValues(nameValues);
return convertObjToString(nameValueList);