[GitHub] metron pull request #1247: METRON-1845 Correct Test Data Load in Elasticsear...

[mmiklavc <gi...@git.apache.org>]

Github user mmiklavc commented on a diff in the pull request:

    https://github.com/apache/metron/pull/1247#discussion_r236440028
  
    --- Diff: metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java ---
    @@ -97,45 +118,63 @@ protected static InMemoryComponent startIndex() throws Exception {
         return es;
       }
     
    -  protected static void loadTestData() throws ParseException, IOException {
    +  protected static void loadTestData() throws Exception {
         ElasticSearchComponent es = (ElasticSearchComponent) indexComponent;
     
    +    // define the bro index template
    +    String broIndex = "bro_index_2017.01.01.01";
         JSONObject broTemplate = JSONUtils.INSTANCE.load(new File(broTemplatePath), JSONObject.class);
         addTestFieldMappings(broTemplate, "bro_doc");
    -    es.getClient().admin().indices().prepareCreate("bro_index_2017.01.01.01")
    -        .addMapping("bro_doc", JSONUtils.INSTANCE.toJSON(broTemplate.get("mappings"), false)).get();
    +    es.getClient().admin().indices().prepareCreate(broIndex)
    +            .addMapping("bro_doc", JSONUtils.INSTANCE.toJSON(broTemplate.get("mappings"), false)).get();
    +
    +    // define the snort index template
    +    String snortIndex = "snort_index_2017.01.01.02";
         JSONObject snortTemplate = JSONUtils.INSTANCE.load(new File(snortTemplatePath), JSONObject.class);
         addTestFieldMappings(snortTemplate, "snort_doc");
    -    es.getClient().admin().indices().prepareCreate("snort_index_2017.01.01.02")
    -        .addMapping("snort_doc", JSONUtils.INSTANCE.toJSON(snortTemplate.get("mappings"), false)).get();
    -
    -    BulkRequestBuilder bulkRequest = es.getClient().prepareBulk()
    -        .setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
    -    JSONArray broArray = (JSONArray) new JSONParser().parse(broData);
    -    for (Object o : broArray) {
    -      JSONObject jsonObject = (JSONObject) o;
    -      IndexRequestBuilder indexRequestBuilder = es.getClient()
    -          .prepareIndex("bro_index_2017.01.01.01", "bro_doc");
    -      indexRequestBuilder = indexRequestBuilder.setId((String) jsonObject.get("guid"));
    -      indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString());
    -      indexRequestBuilder = indexRequestBuilder
    -          .setTimestamp(jsonObject.get("timestamp").toString());
    -      bulkRequest.add(indexRequestBuilder);
    +    es.getClient().admin().indices().prepareCreate(snortIndex)
    +            .addMapping("snort_doc", JSONUtils.INSTANCE.toJSON(snortTemplate.get("mappings"), false)).get();
    +
    +    // setup the classes required to write the test data
    +    AccessConfig accessConfig = createAccessConfig();
    +    ElasticsearchClient client = ElasticsearchUtils.getClient(createGlobalConfig());
    +    ElasticsearchRetrieveLatestDao retrieveLatestDao = new ElasticsearchRetrieveLatestDao(client);
    +    ElasticsearchColumnMetadataDao columnMetadataDao = new ElasticsearchColumnMetadataDao(client);
    +    ElasticsearchRequestSubmitter requestSubmitter = new ElasticsearchRequestSubmitter(client);
    +    ElasticsearchUpdateDao updateDao = new ElasticsearchUpdateDao(client, accessConfig, retrieveLatestDao);
    +    ElasticsearchSearchDao searchDao = new ElasticsearchSearchDao(client, accessConfig, columnMetadataDao, requestSubmitter);
    +
    +    // write the test documents for Bro
    +    List<String> broDocuments = new ArrayList<>();
    +    for (Object broObject: (JSONArray) new JSONParser().parse(broData)) {
    +      broDocuments.add(((JSONObject) broObject).toJSONString());
         }
    -    JSONArray snortArray = (JSONArray) new JSONParser().parse(snortData);
    -    for (Object o : snortArray) {
    -      JSONObject jsonObject = (JSONObject) o;
    -      IndexRequestBuilder indexRequestBuilder = es.getClient()
    -          .prepareIndex("snort_index_2017.01.01.02", "snort_doc");
    -      indexRequestBuilder = indexRequestBuilder.setId((String) jsonObject.get("guid"));
    -      indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString());
    -      indexRequestBuilder = indexRequestBuilder
    -          .setTimestamp(jsonObject.get("timestamp").toString());
    -      bulkRequest.add(indexRequestBuilder);
    +    es.add(updateDao, broIndex, "bro", broDocuments);
    +
    +    // write the test documents for Snort
    +    List<String> snortDocuments = new ArrayList<>();
    +    for (Object snortObject: (JSONArray) new JSONParser().parse(snortData)) {
    +      snortDocuments.add(((JSONObject) snortObject).toJSONString());
         }
    -    BulkResponse bulkResponse = bulkRequest.execute().actionGet();
    -    if (bulkResponse.hasFailures()) {
    -      throw new RuntimeException("Failed to index test data");
    +    es.add(updateDao, snortIndex, "snort", snortDocuments);
    +
    +    // wait until the test documents are visible
    +    assertEventually(() -> Assert.assertEquals(10, findAll(searchDao).getTotal()));
    --- End diff --
    
    @nickwallen is it possible to parameterize the call to do something similar to this? https://github.com/apache/metron/pull/1242/files#diff-99b67fd77500d6232462dc7c27ecff67R148
    
    It's possible this is only available on batch calls, but I wanted to mention it in case we can get away with the assertEventually calls.


---