You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/10/18 10:24:24 UTC

[ambari-infra] branch master updated: AMBARI-24792 - Infra Manager: not all the documents archived (#8)

This is an automated email from the ASF dual-hosted git repository.

oleewere pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ambari-infra.git


The following commit(s) were added to refs/heads/master by this push:
     new e1c57c8  AMBARI-24792 - Infra Manager: not all the documents archived (#8)
e1c57c8 is described below

commit e1c57c854880aae8f7611f8377b250b741ffbb05
Author: kasakrisz <33...@users.noreply.github.com>
AuthorDate: Thu Oct 18 12:24:20 2018 +0200

    AMBARI-24792 - Infra Manager: not all the documents archived (#8)
---
 .../java/org/apache/ambari/infra/S3Client.java     |  9 +++++
 .../ambari/infra/steps/AbstractInfraSteps.java     |  3 +-
 .../apache/ambari/infra/steps/ExportJobsSteps.java | 43 ++++++++++++++++++++--
 .../test/resources/stories/infra_api_tests.story   |  3 ++
 .../ambari/infra/job/archive/SolrProperties.java   |  7 +++-
 .../infra/job/archive/SolrPropertiesTest.java      | 18 +++++++--
 6 files changed, 74 insertions(+), 9 deletions(-)

diff --git a/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/S3Client.java b/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/S3Client.java
index f0b592d..9c8fbcf 100644
--- a/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/S3Client.java
+++ b/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/S3Client.java
@@ -96,4 +96,13 @@ public class S3Client {
       throw new RuntimeException(e);
     }
   }
+
+  public InputStream getObject(String key) {
+    try {
+      return s3client.getObject(bucket, key);
+    }
+    catch (Exception ex) {
+      throw new RuntimeException(ex);
+    }
+  }
 }
diff --git a/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/AbstractInfraSteps.java b/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/AbstractInfraSteps.java
index 24d603b..7a748bc 100644
--- a/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/AbstractInfraSteps.java
+++ b/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/AbstractInfraSteps.java
@@ -114,7 +114,7 @@ public abstract class AbstractInfraSteps {
     }
   }
 
-  protected void addDocument(OffsetDateTime logtime) {
+  protected SolrInputDocument addDocument(OffsetDateTime logtime) {
     SolrInputDocument solrInputDocument = new SolrInputDocument();
     solrInputDocument.addField("logType", "HDFSAudit");
     solrInputDocument.addField("cluster", "cl1");
@@ -148,6 +148,7 @@ public abstract class AbstractInfraSteps {
     solrInputDocument.addField("_ttl_", "+7DAYS");
     solrInputDocument.addField("_expire_at_", "2017-12-15T10:23:19.106Z");
     solr.add(solrInputDocument);
+    return solrInputDocument;
   }
 
   @AfterStories
diff --git a/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/ExportJobsSteps.java b/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/ExportJobsSteps.java
index b1d36d1..e2bbe9d 100644
--- a/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/ExportJobsSteps.java
+++ b/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/ExportJobsSteps.java
@@ -18,6 +18,7 @@
  */
 package org.apache.ambari.infra.steps;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.Objects.requireNonNull;
 import static org.apache.ambari.infra.OffsetDateTimeConverter.SOLR_DATETIME_FORMATTER;
 import static org.apache.ambari.infra.TestUtil.doWithin;
@@ -26,19 +27,24 @@ import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.core.IsCollectionContaining.hasItem;
 import static org.junit.Assert.assertThat;
 
+import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.UncheckedIOException;
 import java.time.Duration;
 import java.time.OffsetDateTime;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.ambari.infra.InfraClient;
 import org.apache.ambari.infra.JobExecutionInfo;
 import org.apache.ambari.infra.S3Client;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
@@ -50,16 +56,21 @@ import org.jbehave.core.annotations.When;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
 public class ExportJobsSteps extends AbstractInfraSteps {
   private static final Logger LOG = LoggerFactory.getLogger(ExportJobsSteps.class);
+  private Set<String> documentIds = new HashSet<>();
 
   private Map<String, JobExecutionInfo> launchedJobs = new HashMap<>();
 
   @Given("$count documents in solr")
   public void addDocuments(int count) {
     OffsetDateTime intervalEnd = OffsetDateTime.now();
+    documentIds.clear();
     for (int i = 0; i < count; ++i) {
-      addDocument(intervalEnd.minusMinutes(i % (count / 10)));
+      documentIds.add(addDocument(intervalEnd.minusMinutes(i % (count / 10))).get("id").getValue().toString());
     }
     getSolr().commit();
   }
@@ -68,13 +79,15 @@ public class ExportJobsSteps extends AbstractInfraSteps {
   public void addDocuments(long count, OffsetDateTime startLogtime, OffsetDateTime endLogtime) {
     Duration duration = Duration.between(startLogtime, endLogtime);
     long increment = duration.toNanos() / count;
-    for (int i = 0; i < count; ++i)
-      addDocument(startLogtime.plusNanos(increment * i));
+    documentIds.clear();
+    for (int i = 0; i < count; ++i) {
+      documentIds.add(addDocument(startLogtime.plusNanos(increment * i)).get("id").getValue().toString());
+    }
     getSolr().commit();
   }
 
   @Given("a file on s3 with key $key")
-  public void addFileToS3(String key) throws Exception {
+  public void addFileToS3(String key) {
     getS3client().putObject(key, "anything".getBytes());
   }
 
@@ -204,4 +217,26 @@ public class ExportJobsSteps extends AbstractInfraSteps {
             .filter(file -> file.getName().contains(text))
             .count(), is(count));
   }
+
+  private static final ObjectMapper json = new ObjectMapper();
+
+  @Then("Check the files $fileNamePart contains the archived documents")
+  public void checkStoredDocumentIds(String fileNamePart) throws Exception {
+    S3Client s3Client = getS3client();
+    int size = documentIds.size();
+    Set<String> storedDocumentIds = new HashSet<>();
+    for (String objectKey : s3Client.listObjectKeys(fileNamePart)) {
+      try (BufferedReader reader = new BufferedReader(new InputStreamReader(new BZip2CompressorInputStream(s3Client.getObject(objectKey)), UTF_8))) {
+        String line;
+        while ((line = reader.readLine()) != null) {
+          Map<String, Object> document = json.readValue(line, new TypeReference<HashMap<String, Object>>() {});
+          String id = document.get("id").toString();
+          storedDocumentIds.add(id);
+          documentIds.remove(id);
+        }
+      }
+    }
+    assertThat(documentIds.size(), is(0));
+    assertThat(storedDocumentIds.size(), is(size));
+  }
 }
diff --git a/ambari-infra-manager-it/src/test/resources/stories/infra_api_tests.story b/ambari-infra-manager-it/src/test/resources/stories/infra_api_tests.story
index 2330474..806dc84 100644
--- a/ambari-infra-manager-it/src/test/resources/stories/infra_api_tests.story
+++ b/ambari-infra-manager-it/src/test/resources/stories/infra_api_tests.story
@@ -11,6 +11,7 @@ Given 10 documents in solr with logtime from 2010-10-09T05:00:00.000Z to 2010-10
 When start archive_audit_logs job with parameters writeBlockSize=3,start=2010-10-09T00:00:00.000Z,end=2010-10-11T00:00:00.000Z after 2 seconds
 Then Check 4 files exists on s3 server with filenames containing the text solr_archive_audit_logs_-_2010-10-09 after 20 seconds
 And solr does not contain documents between 2010-10-09T05:00:00.000Z and 2010-10-09T20:00:00.000Z after 5 seconds
+And Check the files solr_archive_audit_logs_-_2010-10-09 contains the archived documents
 
 
 Scenario: Running archiving job with a bigger start value than end value exports and deletes 0 documents
@@ -32,6 +33,7 @@ When delete file with key solr_archive_audit_logs_-_2011-10-09T08-00-00.000Z.jso
 And restart archive_audit_logs job within 2 seconds
 Then Check 10 files exists on s3 server with filenames containing the text solr_archive_audit_logs_-_2011-10-09 after 20 seconds
 And solr does not contain documents between 2011-10-09T05:00:00.000Z and 2011-10-09T20:00:00.000Z after 5 seconds
+And Check the files solr_archive_audit_logs_-_2011-10-09 contains the archived documents
 
 
 Scenario: After Deleting job deletes documents from solr no document found in the specified interval
@@ -65,3 +67,4 @@ And stop job archive_audit_logs after at least 1 file exists in s3 with filename
 Then Less than 20 files exists on s3 server with filenames containing the text solr_archive_audit_logs_-_2014-03-09 after 20 seconds
 When restart archive_audit_logs job within 10 seconds
 Then Check 25 files exists on s3 server with filenames containing the text solr_archive_audit_logs_-_2014-03-09 after 20 seconds
+And Check the files solr_archive_audit_logs_-_2014-03-09 contains the archived documents
\ No newline at end of file
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrProperties.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrProperties.java
index 1cb2d62..43e871f 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrProperties.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrProperties.java
@@ -94,7 +94,12 @@ public class SolrProperties {
       sortColumns.add(sortValue);
       ++i;
     }
-    solrParameters.setSortColumn(sortColumns.toArray(new String[0]));
+    if (!sortColumns.isEmpty()) {
+      solrParameters.setSortColumn(sortColumns.toArray(new String[0]));
+    }
+    else {
+      solrParameters.setSortColumn(sortColumn);
+    }
 
     return solrParameters;
   }
diff --git a/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrPropertiesTest.java b/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrPropertiesTest.java
index b7bda57..71d25b6 100644
--- a/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrPropertiesTest.java
+++ b/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/SolrPropertiesTest.java
@@ -1,6 +1,7 @@
 package org.apache.ambari.infra.job.archive;
 
 import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.nullValue;
 import static org.junit.Assert.assertThat;
 
 import org.junit.Test;
@@ -27,7 +28,7 @@ import org.springframework.batch.core.JobParametersBuilder;
  */
 public class  SolrPropertiesTest {
   @Test
-  public void testApplySortColumns() {
+  public void testMergeSortColumns() {
     JobParameters jobParameters = new JobParametersBuilder()
             .addString("sortColumn[0]", "logtime")
             .addString("sortColumn[1]", "id")
@@ -42,13 +43,24 @@ public class  SolrPropertiesTest {
   }
 
   @Test
-  public void testApplyWhenNoSortIsDefined() {
+  public void testMergeWhenNoSortIsDefined() {
+    JobParameters jobParameters = new JobParametersBuilder()
+            .toJobParameters();
+
+    SolrProperties solrProperties = new SolrProperties();
+    SolrParameters solrParameters = solrProperties.merge(jobParameters);
+    assertThat(solrParameters.getSortColumn(), is(nullValue()));
+  }
+
+  @Test
+  public void testMergeWhenPropertiesAreDefinedButJobParamsAreNot() {
     JobParameters jobParameters = new JobParametersBuilder()
             .toJobParameters();
 
     SolrProperties solrProperties = new SolrProperties();
     solrProperties.setSortColumn(new String[] {"testColumn"});
     SolrParameters solrParameters = solrProperties.merge(jobParameters);
-    assertThat(solrParameters.getSortColumn().length, is(0));
+    assertThat(solrParameters.getSortColumn().length, is(1));
+    assertThat(solrParameters.getSortColumn()[0], is("testColumn"));
   }
 }
\ No newline at end of file