You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2018/01/05 23:04:08 UTC
[12/50] [abbrv] ambari git commit: AMBARI-22702. Infra Manager:
scheduled deleting of Infra Solr documents (Krisztian Kasa via oleewere )
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85ff23b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryBuilder.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryBuilder.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryBuilder.java
index b3ea14e..0e41169 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryBuilder.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryBuilder.java
@@ -20,27 +20,27 @@ package org.apache.ambari.infra.job.archive;
import org.apache.solr.client.solrj.SolrQuery;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
+import java.util.HashMap;
+import java.util.Map;
import static org.apache.commons.lang.StringUtils.isBlank;
import static org.apache.solr.client.solrj.SolrQuery.ORDER.asc;
public class SolrQueryBuilder {
- public static final Pattern PARAMETER_PATTERN = Pattern.compile("\\$\\{[a-z]+\\}");
-
+ private static final String INTERVAL_START = "start";
+ private static final String INTERVAL_END = "end";
private String queryText;
- private String startValue;
- private String endValue;
+ private final Map<String, String> interval;
private String filterQueryText;
private Document document;
private String[] sortFields;
public SolrQueryBuilder() {
this.queryText = "*:*";
+ interval = new HashMap<>();
+ interval.put(INTERVAL_START, "*");
+ interval.put(INTERVAL_END, "*");
}
public SolrQueryBuilder setQueryText(String queryText) {
@@ -48,14 +48,13 @@ public class SolrQueryBuilder {
return this;
}
- public SolrQueryBuilder setEndValue(String endValue) {
- this.endValue = endValue;
- return this;
- }
-
public SolrQueryBuilder setInterval(String startValue, String endValue) {
- this.startValue = startValue;
- this.endValue = endValue;
+ if (isBlank(startValue))
+ startValue = "*";
+ if (isBlank(endValue))
+ endValue = "*";
+ this.interval.put(INTERVAL_START, startValue);
+ this.interval.put(INTERVAL_END, endValue);
return this;
}
@@ -78,27 +77,17 @@ public class SolrQueryBuilder {
public SolrQuery build() {
SolrQuery solrQuery = new SolrQuery();
- String query = queryText;
- query = setValueOn(query, "${start}", startValue);
- query = setValueOn(query, "${end}", endValue);
-
- solrQuery.setQuery(query);
+ SolrParametrizedString queryText = new SolrParametrizedString(this.queryText).set(interval);
+ solrQuery.setQuery(queryText.toString());
if (filterQueryText != null) {
- String filterQuery = filterQueryText;
- filterQuery = setValueOn(filterQuery, "${start}", startValue);
- filterQuery = setValueOn(filterQuery, "${end}", endValue);
+ SolrParametrizedString filterQuery = new SolrParametrizedString(filterQueryText)
+ .set(interval);
- Set<String> paramNames = collectParamNames(filterQuery);
if (document != null) {
- for (String parameter : paramNames) {
- if (document.get(parameter) != null)
- filterQuery = filterQuery.replace(String.format("${%s}", parameter), String.format("\"%s\"", document.get(parameter)));
- }
+ filterQuery = filterQuery.set(document.getFieldMap());
+ solrQuery.setFilterQueries(filterQuery.toString());
}
-
- if (document == null && paramNames.isEmpty() || document != null && !paramNames.isEmpty())
- solrQuery.setFilterQueries(filterQuery);
}
if (sortFields != null) {
@@ -108,22 +97,4 @@ public class SolrQueryBuilder {
return solrQuery;
}
-
- private String setValueOn(String query, String placeHolder, String value) {
- if (isBlank(value)) {
- value = "*";
- }
- else {
- value = '"' + value + '"';
- }
- return query.replace(placeHolder, value);
- }
-
- private Set<String> collectParamNames(String filterQuery) {
- Matcher matcher = PARAMETER_PATTERN.matcher(filterQuery);
- Set<String> parameters = new HashSet<>();
- while (matcher.find())
- parameters.add(matcher.group().replace("${", "").replace("}", ""));
- return parameters;
- }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85ff23b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/TarGzCompressor.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/TarGzCompressor.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/TarGzCompressor.java
index 55ba58a..8f9d673 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/TarGzCompressor.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/TarGzCompressor.java
@@ -27,7 +27,7 @@ import java.io.*;
public class TarGzCompressor extends AbstractFileAction {
@Override
- public File perform(File inputFile) {
+ public File onPerform(File inputFile) {
File tarGzFile = new File(inputFile.getParent(), inputFile.getName() + ".tar.gz");
try (TarArchiveOutputStream tarArchiveOutputStream = new TarArchiveOutputStream(
new GzipCompressorOutputStream(new FileOutputStream(tarGzFile)))) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85ff23b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/WriteCompletedEvent.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/WriteCompletedEvent.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/WriteCompletedEvent.java
new file mode 100644
index 0000000..49abe22
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/WriteCompletedEvent.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.job.archive;
+
+import java.io.File;
+
+public class WriteCompletedEvent {
+ private final File outFile;
+ private final Document firstDocument;
+ private final Document lastDocument;
+
+ public WriteCompletedEvent(File outFile, Document firstDocument, Document lastDocument) {
+ this.outFile = outFile;
+ this.firstDocument = firstDocument;
+ this.lastDocument = lastDocument;
+ }
+
+ public File getOutFile() {
+ return outFile;
+ }
+
+ public Document getFirstDocument() {
+ return firstDocument;
+ }
+
+ public Document getLastDocument() {
+ return lastDocument;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85ff23b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingConfiguration.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingConfiguration.java
new file mode 100644
index 0000000..4fce4b6
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingConfiguration.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.job.deleting;
+
+import org.apache.ambari.infra.job.JobPropertyMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.JobScope;
+import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
+import org.springframework.batch.core.configuration.annotation.StepScope;
+import org.springframework.batch.core.configuration.support.JobRegistryBeanPostProcessor;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import javax.annotation.PostConstruct;
+import javax.inject.Inject;
+
+@Configuration
+public class DocumentDeletingConfiguration {
+ private static final Logger LOG = LoggerFactory.getLogger(DocumentDeletingConfiguration.class);
+
+ @Inject
+ private DocumentDeletingPropertyMap propertyMap;
+
+ @Inject
+ private StepBuilderFactory steps;
+
+ @Inject
+ private JobBuilderFactory jobs;
+
+ @Inject
+ private JobRegistryBeanPostProcessor jobRegistryBeanPostProcessor;
+
+ @Inject
+ @Qualifier("deleteStep")
+ private Step deleteStep;
+
+ @PostConstruct
+ public void createJobs() {
+ propertyMap.getSolrDataDeleting().values().forEach(DocumentDeletingProperties::validate);
+
+ propertyMap.getSolrDataDeleting().keySet().forEach(jobName -> {
+ LOG.info("Registering data deleting job {}", jobName);
+ Job job = logDeleteJob(jobName, deleteStep);
+ jobRegistryBeanPostProcessor.postProcessAfterInitialization(job, jobName);
+ });
+ }
+
+ private Job logDeleteJob(String jobName, Step logExportStep) {
+ return jobs.get(jobName).listener(new JobPropertyMap<>(propertyMap)).start(logExportStep).build();
+ }
+
+ @Bean
+ @JobScope
+ public Step deleteStep(DocumentWiperTasklet tasklet) {
+ return steps.get("delete")
+ .tasklet(tasklet)
+ .build();
+ }
+
+ @Bean
+ @StepScope
+ public DocumentWiperTasklet documentWiperTasklet(
+ @Value("#{stepExecution.jobExecution.executionContext.get('jobProperties')}") DocumentDeletingProperties properties,
+ @Value("#{jobParameters[start]}") String start,
+ @Value("#{jobParameters[end]}") String end) {
+ return new DocumentWiperTasklet(properties, start, end);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85ff23b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingProperties.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingProperties.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingProperties.java
new file mode 100644
index 0000000..63b7dd2
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingProperties.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.job.deleting;
+
+import org.apache.ambari.infra.job.JobProperties;
+import org.springframework.batch.core.JobParameters;
+
+import static org.apache.commons.lang.StringUtils.isBlank;
+
+public class DocumentDeletingProperties extends JobProperties<DocumentDeletingProperties> {
+ private String zooKeeperConnectionString;
+ private String collection;
+ private String filterField;
+
+ public DocumentDeletingProperties() {
+ super(DocumentDeletingProperties.class);
+ }
+
+ public String getZooKeeperConnectionString() {
+ return zooKeeperConnectionString;
+ }
+
+ public void setZooKeeperConnectionString(String zooKeeperConnectionString) {
+ this.zooKeeperConnectionString = zooKeeperConnectionString;
+ }
+
+ public String getCollection() {
+ return collection;
+ }
+
+ public void setCollection(String collection) {
+ this.collection = collection;
+ }
+
+ public String getFilterField() {
+ return filterField;
+ }
+
+ public void setFilterField(String filterField) {
+ this.filterField = filterField;
+ }
+
+ @Override
+ public void apply(JobParameters jobParameters) {
+ zooKeeperConnectionString = jobParameters.getString("zooKeeperConnectionString", zooKeeperConnectionString);
+ collection = jobParameters.getString("collection", collection);
+ filterField = jobParameters.getString("filterField", filterField);
+ }
+
+ @Override
+ public void validate() {
+ if (isBlank(zooKeeperConnectionString))
+ throw new IllegalArgumentException("The property zooKeeperConnectionString can not be null or empty string!");
+
+ if (isBlank(collection))
+ throw new IllegalArgumentException("The property collection can not be null or empty string!");
+
+ if (isBlank(filterField))
+ throw new IllegalArgumentException("The property filterField can not be null or empty string!");
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85ff23b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingPropertyMap.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingPropertyMap.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingPropertyMap.java
new file mode 100644
index 0000000..fccfd59
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingPropertyMap.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.job.deleting;
+
+import org.apache.ambari.infra.job.PropertyMap;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+import java.util.Map;
+
+@Configuration
+@ConfigurationProperties(prefix = "infra-manager.jobs")
+public class DocumentDeletingPropertyMap implements PropertyMap<DocumentDeletingProperties> {
+ private Map<String, DocumentDeletingProperties> solrDataDeleting;
+
+ public Map<String, DocumentDeletingProperties> getSolrDataDeleting() {
+ return solrDataDeleting;
+ }
+
+ public void setSolrDataDeleting(Map<String, DocumentDeletingProperties> solrDataDeleting) {
+ this.solrDataDeleting = solrDataDeleting;
+ }
+
+ @Override
+ public Map<String, DocumentDeletingProperties> getPropertyMap() {
+ return getSolrDataDeleting();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85ff23b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentWiperTasklet.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentWiperTasklet.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentWiperTasklet.java
new file mode 100644
index 0000000..463e6e0
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentWiperTasklet.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.job.deleting;
+
+import org.apache.ambari.infra.job.SolrDAOBase;
+import org.apache.solr.client.solrj.util.ClientUtils;
+import org.springframework.batch.core.StepContribution;
+import org.springframework.batch.core.scope.context.ChunkContext;
+import org.springframework.batch.core.step.tasklet.Tasklet;
+import org.springframework.batch.repeat.RepeatStatus;
+
+public class DocumentWiperTasklet extends SolrDAOBase implements Tasklet {
+ private final String filterField;
+ private final String start;
+ private final String end;
+
+ public DocumentWiperTasklet(DocumentDeletingProperties properties, String start, String end) {
+ super(properties.getZooKeeperConnectionString(), properties.getCollection());
+ this.filterField = properties.getFilterField();
+ this.start = start;
+ this.end = end;
+ }
+
+ @Override
+ public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) {
+ delete(String.format("%s:[%s TO %s]", filterField, getValue(start), getValue(end)));
+ return RepeatStatus.FINISHED;
+ }
+
+ private String getValue(String value) {
+ return "*".equals(value) ? value : ClientUtils.escapeQueryChars(value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85ff23b/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager.properties
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager.properties b/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager.properties
index 27b36b3..9103d09 100644
--- a/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager.properties
+++ b/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager.properties
@@ -20,33 +20,34 @@ management.security.enabled=false
management.health.solr.enabled=false
infra-manager.server.data.folder=/tmp
-infra-manager.jobs.solr_data_export.export_service_logs.zoo_keeper_connection_string=zookeeper:2181
+infra-manager.jobs.solr_data_export.export_service_logs.solr.zoo_keeper_connection_string=zookeeper:2181
+infra-manager.jobs.solr_data_export.export_service_logs.solr.collection=hadoop_logs
+infra-manager.jobs.solr_data_export.export_service_logs.solr.query_text=logtime:[${start} TO ${end}]
+infra-manager.jobs.solr_data_export.export_service_logs.solr.filter_query_text=(logtime:${logtime} AND id:{${id} TO *]) OR logtime:{${logtime} TO ${end}]
+infra-manager.jobs.solr_data_export.export_service_logs.solr.sort_column[0]=logtime
+infra-manager.jobs.solr_data_export.export_service_logs.solr.sort_column[1]=id
infra-manager.jobs.solr_data_export.export_service_logs.read_block_size=100
infra-manager.jobs.solr_data_export.export_service_logs.write_block_size=150
infra-manager.jobs.solr_data_export.export_service_logs.file_name_suffix_column=logtime
+infra-manager.jobs.solr_data_export.export_service_logs.file_name_suffix_date_format=yyyy-MM-dd'T'HH-mm-ss.SSSX
infra-manager.jobs.solr_data_export.export_service_logs.destination_directory_path=/tmp/ambariInfraManager
-infra-manager.jobs.solr_data_export.export_service_logs.query.collection=hadoop_logs
-infra-manager.jobs.solr_data_export.export_service_logs.query.query_text=logtime:[${start} TO ${end}]
-infra-manager.jobs.solr_data_export.export_service_logs.query.filter_query_text=(logtime:${logtime} AND id:{${id} TO *]) OR logtime:{${logtime} TO ${end}]
-infra-manager.jobs.solr_data_export.export_service_logs.query.sort_column[0]=logtime
-infra-manager.jobs.solr_data_export.export_service_logs.query.sort_column[1]=id
-infra-manager.jobs.solr_data_export.export_audit_logs.zoo_keeper_connection_string=zookeeper:2181
-infra-manager.jobs.solr_data_export.export_audit_logs.read_block_size=100
-infra-manager.jobs.solr_data_export.export_audit_logs.write_block_size=150
+infra-manager.jobs.solr_data_export.archive_audit_logs.solr.zoo_keeper_connection_string=zookeeper:2181
+infra-manager.jobs.solr_data_export.archive_audit_logs.solr.collection=audit_logs
+infra-manager.jobs.solr_data_export.archive_audit_logs.solr.query_text=logtime:[${start} TO ${end}]
+infra-manager.jobs.solr_data_export.archive_audit_logs.solr.filter_query_text=(logtime:${logtime} AND id:{${id} TO *]) OR logtime:{${logtime} TO ${end}]
+infra-manager.jobs.solr_data_export.archive_audit_logs.solr.sort_column[0]=logtime
+infra-manager.jobs.solr_data_export.archive_audit_logs.solr.sort_column[1]=id
+infra-manager.jobs.solr_data_export.archive_audit_logs.solr.delete_query_text=logtime:[${start.logtime} TO ${end.logtime}} OR (logtime:${end.logtime} AND id:[* TO ${end.id}])
+infra-manager.jobs.solr_data_export.archive_audit_logs.read_block_size=100
+infra-manager.jobs.solr_data_export.archive_audit_logs.write_block_size=150
# TODO: logtime may not be enough: The same filename can be generated when more than write_block_size count docs has the same logtime value
-infra-manager.jobs.solr_data_export.export_audit_logs.file_name_suffix_column=logtime
-infra-manager.jobs.solr_data_export.export_audit_logs.destination_directory_path=/tmp/ambariInfraManager
-infra-manager.jobs.solr_data_export.export_audit_logs.query.collection=audit_logs
-infra-manager.jobs.solr_data_export.export_audit_logs.query.query_text=logtime:[${start} TO ${end}]
-infra-manager.jobs.solr_data_export.export_audit_logs.query.filter_query_text=(logtime:${logtime} AND id:{${id} TO *]) OR logtime:{${logtime} TO ${end}]
-infra-manager.jobs.solr_data_export.export_audit_logs.query.sort_column[0]=logtime
-infra-manager.jobs.solr_data_export.export_audit_logs.query.sort_column[1]=id
-# TODO: s3_access_key and s3_secret_key to separate file
-infra-manager.jobs.solr_data_export.export_audit_logs.s3_access_key=remote-identity
-infra-manager.jobs.solr_data_export.export_audit_logs.s3_secret_key=remote-credential
-infra-manager.jobs.solr_data_export.export_audit_logs.s3_key_prefix=solr_archive_
-infra-manager.jobs.solr_data_export.export_audit_logs.s3_bucket_name=testbucket
-infra-manager.jobs.solr_data_export.export_audit_logs.s3_endpoint=http://fakes3:4569
+infra-manager.jobs.solr_data_export.archive_audit_logs.file_name_suffix_column=logtime
+infra-manager.jobs.solr_data_export.archive_audit_logs.file_name_suffix_date_format=yyyy-MM-dd'T'HH-mm-ss.SSSX
+infra-manager.jobs.solr_data_export.archive_audit_logs.destination_directory_path=/tmp/ambariInfraManager
+#infra-manager.jobs.solr_data_export.archive_audit_logs.s3_access_file=<any>.csv
+infra-manager.jobs.solr_data_export.archive_audit_logs.s3_key_prefix=solr_archive_
+infra-manager.jobs.solr_data_export.archive_audit_logs.s3_bucket_name=testbucket
+infra-manager.jobs.solr_data_export.archive_audit_logs.s3_endpoint=http://fakes3:4569
# TODO: configure ranger audit logs
#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.zoo_keeper_connection_string=zookeeper:2181
#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.read_block_size=100
@@ -58,3 +59,6 @@ infra-manager.jobs.solr_data_export.export_audit_logs.s3_endpoint=http://fakes3:
#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.query.filter_query_text=(logtime:"${logtime}" AND id:{"${id}" TO *]) OR logtime:{"${logtime}" TO "${end}"]
#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.query.sort_column[0]=logtime
#infra-manager.jobs.solr_data_export.export_ranger_audit_logs.query.sort_column[1]=id
+infra-manager.jobs.solr_data_deleting.delete_audit_logs.zoo_keeper_connection_string=zookeeper:2181
+infra-manager.jobs.solr_data_deleting.delete_audit_logs.collection=audit_logs
+infra-manager.jobs.solr_data_deleting.delete_audit_logs.filter_field=logtime
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85ff23b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/JobPropertiesTest.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/JobPropertiesTest.java b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/JobPropertiesTest.java
new file mode 100644
index 0000000..6a76229
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/JobPropertiesTest.java
@@ -0,0 +1,56 @@
+package org.apache.ambari.infra.job;
+
+import org.apache.ambari.infra.job.archive.DocumentExportProperties;
+import org.apache.ambari.infra.job.archive.SolrProperties;
+import org.junit.Test;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+public class JobPropertiesTest {
+ @Test
+ public void testDeepCopy() throws Exception {
+ DocumentExportProperties documentExportProperties = new DocumentExportProperties();
+ documentExportProperties.setDestinationDirectoryPath("/tmp");
+ documentExportProperties.setFileNameSuffixColumn(".json");
+ documentExportProperties.setReadBlockSize(10);
+ documentExportProperties.setWriteBlockSize(20);
+ SolrProperties solr = new SolrProperties();
+ solr.setZooKeeperConnectionString("localhost:2181");
+ solr.setFilterQueryText("id:1167");
+ solr.setQueryText("name:'Joe'");
+ solr.setCollection("Users");
+ solr.setSortColumn(new String[] {"name"});
+ documentExportProperties.setSolr(solr);
+
+ DocumentExportProperties parsed = documentExportProperties.deepCopy();
+
+ assertThat(parsed.getDestinationDirectoryPath(), is(documentExportProperties.getDestinationDirectoryPath()));
+ assertThat(parsed.getFileNameSuffixColumn(), is(documentExportProperties.getFileNameSuffixColumn()));
+ assertThat(parsed.getReadBlockSize(), is(documentExportProperties.getReadBlockSize()));
+ assertThat(parsed.getWriteBlockSize(), is(documentExportProperties.getWriteBlockSize()));
+ assertThat(parsed.getSolr().getZooKeeperConnectionString(), is(documentExportProperties.getSolr().getZooKeeperConnectionString()));
+ assertThat(parsed.getSolr().getQueryText(), is(solr.getQueryText()));
+ assertThat(parsed.getSolr().getFilterQueryText(), is(solr.getFilterQueryText()));
+ assertThat(parsed.getSolr().getCollection(), is(solr.getCollection()));
+ assertThat(parsed.getSolr().getSortColumn(), is(solr.getSortColumn()));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85ff23b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/DocumentExportPropertiesTest.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/DocumentExportPropertiesTest.java b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/DocumentExportPropertiesTest.java
deleted file mode 100644
index ae93710..0000000
--- a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/DocumentExportPropertiesTest.java
+++ /dev/null
@@ -1,54 +0,0 @@
-package org.apache.ambari.infra.job.archive;
-
-import org.junit.Test;
-
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.assertThat;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-public class DocumentExportPropertiesTest {
- @Test
- public void testDeepCopy() throws Exception {
- DocumentExportProperties documentExportProperties = new DocumentExportProperties();
- documentExportProperties.setDestinationDirectoryPath("/tmp");
- documentExportProperties.setFileNameSuffixColumn(".json");
- documentExportProperties.setReadBlockSize(10);
- documentExportProperties.setWriteBlockSize(20);
- documentExportProperties.setZooKeeperConnectionString("localhost:2181");
- SolrQueryProperties query = new SolrQueryProperties();
- query.setFilterQueryText("id:1167");
- query.setQueryText("name:'Joe'");
- query.setCollection("Users");
- query.setSortColumn(new String[] {"name"});
- documentExportProperties.setQuery(query);
-
- DocumentExportProperties parsed = documentExportProperties.deepCopy();
-
- assertThat(parsed.getDestinationDirectoryPath(), is(documentExportProperties.getDestinationDirectoryPath()));
- assertThat(parsed.getFileNameSuffixColumn(), is(documentExportProperties.getFileNameSuffixColumn()));
- assertThat(parsed.getReadBlockSize(), is(documentExportProperties.getReadBlockSize()));
- assertThat(parsed.getWriteBlockSize(), is(documentExportProperties.getWriteBlockSize()));
- assertThat(parsed.getZooKeeperConnectionString(), is(documentExportProperties.getZooKeeperConnectionString()));
- assertThat(parsed.getQuery().getQueryText(), is(query.getQueryText()));
- assertThat(parsed.getQuery().getFilterQueryText(), is(query.getFilterQueryText()));
- assertThat(parsed.getQuery().getCollection(), is(query.getCollection()));
- assertThat(parsed.getQuery().getSortColumn(), is(query.getSortColumn()));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85ff23b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/FileNameSuffixFormatterTest.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/FileNameSuffixFormatterTest.java b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/FileNameSuffixFormatterTest.java
new file mode 100644
index 0000000..34e679f
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/FileNameSuffixFormatterTest.java
@@ -0,0 +1,58 @@
+package org.apache.ambari.infra.job.archive;
+
+import org.junit.Test;
+
+import java.util.HashMap;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+public class FileNameSuffixFormatterTest {
+
+ private FileNameSuffixFormatter formatter = new FileNameSuffixFormatter("logtime", "yyyy-MM-dd'T'hh-mm-ss-SSSX");
+
+ @Test(expected = NullPointerException.class)
+ public void testFormatWhenDocumentIsNullThrowingException() throws Exception {
+ formatter.format(null);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testFormatWhenSpecifiedColumnDoesNotExistsInTheDocumentThrowingException() throws Exception {
+ formatter.format(new Document(new HashMap<>()));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testFormatWhenSpecifiedColumnContainsBlankValueThrowingException() throws Exception {
+ formatter.format(new Document(new HashMap<String, String>() {{ put("logtime", " "); }}));
+ }
+
+ @Test
+ public void testFormatWhenNoDateFormatSpecifiedRawColumnValueReturned() throws Exception {
+ FileNameSuffixFormatter formatter = new FileNameSuffixFormatter("logtime", null);
+ assertThat(formatter.format(new Document(new HashMap<String, String>() {{ put("logtime", "Monday"); }})), is("Monday"));
+ }
+
+ @Test
+ public void testFormatWhenDateFormatIsSpecifiedAFormattedValueReturned() throws Exception {
+ assertThat(formatter.format(new Document(new HashMap<String, String>() {{ put("logtime", "2017-12-15T10:12:33.453Z"); }})), is("2017-12-15T10-12-33-453Z"));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85ff23b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/LocalDocumentItemWriterTest.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/LocalDocumentItemWriterTest.java b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/LocalDocumentItemWriterTest.java
index 3af93bc..85e79e1 100644
--- a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/LocalDocumentItemWriterTest.java
+++ b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/LocalDocumentItemWriterTest.java
@@ -32,10 +32,13 @@ import org.junit.runner.RunWith;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
+import static org.easymock.EasyMock.cmp;
import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.LogicalOperator.EQUAL;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;
@@ -44,6 +47,7 @@ public class LocalDocumentItemWriterTest extends EasyMockSupport {
private static final Document DOCUMENT = new Document(new HashMap<String, String>() {{ put("id", "1"); }});
private static final Document DOCUMENT2 = new Document(new HashMap<String, String>() {{ put("id", "2"); }});
+ private static final Document DOCUMENT3 = new Document(new HashMap<String, String>() {{ put("id", "3"); }});
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private LocalDocumentItemWriter localDocumentItemWriter;
@@ -65,17 +69,30 @@ public class LocalDocumentItemWriterTest extends EasyMockSupport {
@Test
public void testWrite() throws Exception {
- itemWriterListener.onCompleted(outFile); expectLastCall();
+ itemWriterListener.onCompleted(
+ cmp(new WriteCompletedEvent(outFile, DOCUMENT, DOCUMENT3), writeCompletedEventEqualityComparator(), EQUAL)); expectLastCall();
replayAll();
localDocumentItemWriter.write(DOCUMENT);
localDocumentItemWriter.write(DOCUMENT2);
+ localDocumentItemWriter.write(DOCUMENT3);
localDocumentItemWriter.close();
List<Document> documentList = readBack(outFile);
- assertThat(documentList.size(), is(2));
+ assertThat(documentList.size(), is(3));
assertThat(documentList.get(0).get("id"), is(DOCUMENT.get("id")));
assertThat(documentList.get(1).get("id"), is(DOCUMENT2.get("id")));
+ assertThat(documentList.get(2).get("id"), is(DOCUMENT3.get("id")));
+ }
+
+ private Comparator<WriteCompletedEvent> writeCompletedEventEqualityComparator() {
+ return (o1, o2) -> {
+ if (o1.getOutFile().equals(o2.getOutFile()) &&
+ o1.getFirstDocument().equals(o2.getFirstDocument()) &&
+ o1.getLastDocument().equals(o2.getLastDocument()))
+ return 0;
+ return 1;
+ };
}
private List<Document> readBack(File file) throws IOException {
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85ff23b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrParametrizedStringTest.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrParametrizedStringTest.java b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrParametrizedStringTest.java
new file mode 100644
index 0000000..018c993
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrParametrizedStringTest.java
@@ -0,0 +1,57 @@
+package org.apache.ambari.infra.job.archive;
+
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+public class SolrParametrizedStringTest {
+
+ private static final Map<String, String> PARAMETERS_1 = new HashMap<String, String>() {{ put("id", "1"); put("name", "User"); put("product", "Computer"); }};
+ private static final Map<String, String> PARAMETERS_START = new HashMap<String, String>() {{ put("price", "1000"); }};
+ private static final Map<String, String> PARAMETERS_END = new HashMap<String, String>() {{ put("price", "2000"); }};
+
+ @Test
+ public void testToStringEmptyStringResultsEmptyString() {
+ assertThat(new SolrParametrizedString("").set(PARAMETERS_1).toString(), is(""));
+ }
+
+ @Test
+ public void testParameterlessStringResultsItself() {
+ assertThat(new SolrParametrizedString("Hello World!").set(PARAMETERS_1).toString(), is("Hello World!"));
+ }
+
+ @Test
+ public void testParametersAreReplacedIfFoundInString() {
+ assertThat(new SolrParametrizedString("Hello ${name}!").set(PARAMETERS_1).toString(), is("Hello User!"));
+ }
+
+ @Test
+ public void testWhenStringContainsPrefixedParamtersOnlyPrefixedParametersAreSet() {
+ assertThat(new SolrParametrizedString("The ${product} price is between $${start.price} and $${end.price}.")
+ .set(PARAMETERS_1)
+ .set("start", PARAMETERS_START)
+ .set("end", PARAMETERS_END).toString(), is("The Computer price is between $1000 and $2000."));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85ff23b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrPropertiesTest.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrPropertiesTest.java b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrPropertiesTest.java
new file mode 100644
index 0000000..be8a226
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrPropertiesTest.java
@@ -0,0 +1,54 @@
+package org.apache.ambari.infra.job.archive;
+
+import org.junit.Test;
+import org.springframework.batch.core.JobParameters;
+import org.springframework.batch.core.JobParametersBuilder;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+public class SolrPropertiesTest {
+ @Test
+ public void testApplySortColumns() throws Exception {
+ JobParameters jobParameters = new JobParametersBuilder()
+ .addString("sortColumn[0]", "logtime")
+ .addString("sortColumn[1]", "id")
+ .toJobParameters();
+
+ SolrProperties solrProperties = new SolrProperties();
+ solrProperties.setSortColumn(new String[] {"testColumn"});
+ solrProperties.apply(jobParameters);
+ assertThat(solrProperties.getSortColumn().length, is(2));
+ assertThat(solrProperties.getSortColumn()[0], is("logtime"));
+ assertThat(solrProperties.getSortColumn()[1], is("id"));
+ }
+
+ @Test
+ public void testApplyWhenNoSortIsDefined() throws Exception {
+ JobParameters jobParameters = new JobParametersBuilder()
+ .toJobParameters();
+
+ SolrProperties solrProperties = new SolrProperties();
+ solrProperties.setSortColumn(new String[] {"testColumn"});
+ solrProperties.apply(jobParameters);
+ assertThat(solrProperties.getSortColumn().length, is(1));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/a85ff23b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrQueryBuilderTest.java
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrQueryBuilderTest.java b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrQueryBuilderTest.java
index e9513dc..ee08279 100644
--- a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrQueryBuilderTest.java
+++ b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrQueryBuilderTest.java
@@ -21,15 +21,10 @@ package org.apache.ambari.infra.job.archive;
import org.apache.solr.client.solrj.SolrQuery;
import org.junit.Test;
-import java.util.ArrayList;
import java.util.HashMap;
-import java.util.List;
-import java.util.regex.Matcher;
-import static org.apache.ambari.infra.job.archive.SolrQueryBuilder.PARAMETER_PATTERN;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.nullValue;
-import static org.hamcrest.Matchers.hasSize;
import static org.junit.Assert.assertThat;
public class SolrQueryBuilderTest {
@@ -46,40 +41,40 @@ public class SolrQueryBuilderTest {
}
@Test
- public void testSetQuery() throws Exception {
+ public void testSetQueryReplacesTheDefaultQueryTextAndParameterPlaceholdersAreReplacedToValues() throws Exception {
SolrQuery solrQuery = new SolrQueryBuilder()
.setQueryText("logtime:[* TO ${end}]")
- .setEndValue("2017-11-27'T'10:12:11.372Z")
+ .setInterval(null, "2017-11-27'T'10:12:11.372Z")
.build();
- assertThat(solrQuery.getQuery(), is("logtime:[* TO \"2017-11-27'T'10:12:11.372Z\"]"));
+ assertThat(solrQuery.getQuery(), is("logtime:[* TO 2017\\-11\\-27'T'10\\:12\\:11.372Z]"));
}
@Test
- public void testSetFilterQuery() throws Exception {
+ public void testSetFilterQueryAddsAFilterQueryAndParameterPlaceholdersAreReplacedToValues() throws Exception {
SolrQuery solrQuery = new SolrQueryBuilder()
.setFilterQueryText("(logtime:${logtime} AND id:{${id} TO *]) OR logtime:{${logtime} TO ${end}]")
.setDocument(DOCUMENT)
- .setEndValue("2017-11-27'T'10:12:11.372Z")
+ .setInterval(null, "2017-11-27'T'10:12:11.372Z")
.build();
- assertThat(solrQuery.getFilterQueries()[0], is("(logtime:\"2017-10-02'T'10:00:11.634Z\" AND id:{\"1\" TO *]) OR logtime:{\"2017-10-02'T'10:00:11.634Z\" TO \"2017-11-27'T'10:12:11.372Z\"]"));
+ assertThat(solrQuery.getFilterQueries()[0], is( "(logtime:2017\\-10\\-02'T'10\\:00\\:11.634Z AND id:{1 TO *]) OR logtime:{2017\\-10\\-02'T'10\\:00\\:11.634Z TO 2017\\-11\\-27'T'10\\:12\\:11.372Z]"));
}
@Test
public void testSetFilterQueryWhenDocumentIsNull() throws Exception {
SolrQuery solrQuery = new SolrQueryBuilder()
.setFilterQueryText("(logtime:\"${logtime}\" AND id:{\"${id}\" TO *]) OR logtime:{\"${logtime}\" TO \"${end}\"]")
- .setEndValue("2017-11-27'T'10:12:11.372Z")
+ .setInterval(null, "2017-11-27'T'10:12:11.372Z")
.build();
assertThat(solrQuery.getFilterQueries(), is(nullValue()));
}
@Test
- public void testSetFilterQueryWhenEndValueIsNull() throws Exception {
+ public void testNullEndValueDoesNotAffectFilterQuery() throws Exception {
SolrQuery solrQuery = new SolrQueryBuilder()
.setFilterQueryText("logtime:${logtime} AND id:{${id} TO *]")
.setDocument(DOCUMENT)
.build();
- assertThat(solrQuery.getFilterQueries()[0], is("logtime:\"2017-10-02'T'10:00:11.634Z\" AND id:{\"1\" TO *]"));
+ assertThat(solrQuery.getFilterQueries()[0], is("logtime:2017\\-10\\-02'T'10\\:00\\:11.634Z AND id:{1 TO *]"));
}
@Test
@@ -91,20 +86,6 @@ public class SolrQueryBuilderTest {
}
@Test
- public void testRegex() throws Exception {
- Matcher matcher = PARAMETER_PATTERN.matcher("(logtime:\"${logtime}\" AND id:{\"${id}\" TO *]) OR logtime:{\"${logtime}\" TO \"${end}\"]");
- List<String> parameters = new ArrayList<>();
- while (matcher.find())
- parameters.add(matcher.group());
-
- assertThat(parameters, hasSize(4));
- assertThat(parameters.get(0), is("${logtime}"));
- assertThat(parameters.get(1), is("${id}"));
- assertThat(parameters.get(2), is("${logtime}"));
- assertThat(parameters.get(3), is("${end}"));
- }
-
- @Test
public void testSort() throws Exception {
SolrQuery solrQuery = new SolrQueryBuilder().addSort("logtime", "id").build();
assertThat(solrQuery.getSorts().get(0).getItem(), is("logtime"));
@@ -113,7 +94,7 @@ public class SolrQueryBuilderTest {
@Test
public void test_start_and_end_values_are_given() throws Exception {
- SolrQuery solrQuery = new SolrQueryBuilder().setQueryText("id:[${start} TO ${end}]").setInterval("10", "13").build();
+ SolrQuery solrQuery = new SolrQueryBuilder().setQueryText("id:[\"${start}\" TO \"${end}\"]").setInterval("10", "13").build();
assertThat(solrQuery.getQuery(), is("id:[\"10\" TO \"13\"]"));
}