You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2018/07/11 01:32:32 UTC
[16/50] [abbrv] metron git commit: METRON-1547 Solr Comment Fields
(justinleet) closes apache/metron#1037
METRON-1547 Solr Comment Fields (justinleet) closes apache/metron#1037
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a68d031b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a68d031b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a68d031b
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: a68d031b01fe677d84abb0d25aca4f2ceaf90c53
Parents: 9348c60
Author: justinleet <ju...@gmail.com>
Authored: Tue Jun 5 14:59:29 2018 -0400
Committer: leet <le...@apache.org>
Committed: Tue Jun 5 14:59:29 2018 -0400
----------------------------------------------------------------------
.../alert-details/alert-details.component.ts | 25 ++-
.../src/app/model/comment-add-remove-request.ts | 25 +++
.../src/app/service/update.service.ts | 27 +++
.../rest/controller/UpdateController.java | 23 +++
.../metron/rest/service/UpdateService.java | 3 +
.../rest/service/impl/UpdateServiceImpl.java | 19 +++
.../UpdateControllerIntegrationTest.java | 92 ++++++++--
.../elasticsearch/dao/ElasticsearchDao.java | 21 +++
.../dao/ElasticsearchMetaAlertDao.java | 22 +++
.../dao/ElasticsearchMetaAlertUpdateDao.java | 23 +++
.../dao/ElasticsearchUpdateDao.java | 70 ++++++++
.../dao/ElasticsearchMetaAlertDaoTest.java | 17 ++
.../ElasticsearchUpdateIntegrationTest.java | 3 +-
.../apache/metron/indexing/dao/HBaseDao.java | 93 +++++++++-
.../apache/metron/indexing/dao/IndexDao.java | 2 +
.../metron/indexing/dao/MultiIndexDao.java | 47 ++++++
.../indexing/dao/search/AlertComment.java | 130 ++++++++++++++
.../dao/update/CommentAddRemoveRequest.java | 78 +++++++++
.../metron/indexing/dao/update/Document.java | 14 +-
.../metron/indexing/dao/update/PatchUtil.java | 50 ------
.../metron/indexing/dao/update/UpdateDao.java | 33 +++-
.../apache/metron/indexing/dao/InMemoryDao.java | 17 ++
.../indexing/dao/InMemoryMetaAlertDao.java | 17 ++
.../indexing/dao/UpdateIntegrationTest.java | 169 ++++++++++++++++++-
.../AbstractLuceneMetaAlertUpdateDaoTest.java | 17 ++
.../integration/HBaseDaoIntegrationTest.java | 79 ++++++++-
.../src/main/config/schema/bro/schema.xml | 3 +
.../src/main/config/schema/snort/schema.xml | 3 +
.../src/main/config/schema/yaf/schema.xml | 3 +
.../org/apache/metron/solr/dao/SolrDao.java | 41 ++++-
.../metron/solr/dao/SolrMetaAlertDao.java | 21 +++
.../metron/solr/dao/SolrMetaAlertUpdateDao.java | 23 +++
.../metron/solr/dao/SolrRetrieveLatestDao.java | 1 +
.../apache/metron/solr/dao/SolrSearchDao.java | 17 ++
.../apache/metron/solr/dao/SolrUpdateDao.java | 114 ++++++++++++-
.../apache/metron/solr/dao/SolrUtilities.java | 36 +++-
.../org/apache/metron/solr/dao/SolrDaoTest.java | 5 +-
.../metron/solr/dao/SolrMetaAlertDaoTest.java | 18 +-
.../metron/solr/dao/SolrUpdateDaoTest.java | 94 +++++++++--
.../integration/SolrSearchIntegrationTest.java | 6 +-
.../integration/SolrUpdateIntegrationTest.java | 24 ++-
.../resources/config/test/conf/managed-schema | 3 +
42 files changed, 1400 insertions(+), 128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
index e1c1685..6a07e08 100644
--- a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
+++ b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
@@ -30,6 +30,7 @@ import {AlertComment} from './alert-comment';
import {AuthenticationService} from '../../service/authentication.service';
import {MetronDialogBox} from '../../shared/metron-dialog-box';
import {META_ALERTS_INDEX, META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
+import {CommentAddRemoveRequest} from "../../model/comment-add-remove-request";
export enum AlertState {
NEW, OPEN, ESCALATE, DISMISS, RESOLVE
@@ -204,10 +205,15 @@ export class AlertDetailsComponent implements OnInit {
}
onAddComment() {
- let alertComment = new AlertComment(this.alertCommentStr, this.authenticationService.getCurrentUserName(), new Date().getTime());
- let tAlertComments = this.alertCommentsWrapper.map(alertsWrapper => alertsWrapper.alertComment);
- tAlertComments.unshift(alertComment);
- this.patchAlert(new Patch('add', '/comments', tAlertComments));
+ let commentRequest = new CommentAddRemoveRequest();
+ commentRequest.guid = this.alertSource.guid;
+ commentRequest.comment = this.alertCommentStr;
+ commentRequest.username = this.authenticationService.getCurrentUserName();
+ commentRequest.timestamp = new Date().getTime();
+ commentRequest.sensorType = this.alertSourceType;
+ this.updateService.addComment(commentRequest).subscribe( () => {
+ this.getData(true);
+ });
}
patchAlert(patch: Patch) {
@@ -232,8 +238,15 @@ export class AlertDetailsComponent implements OnInit {
this.metronDialogBox.showConfirmationMessage(commentText).subscribe(response => {
if (response) {
- this.alertCommentsWrapper.splice(index, 1);
- this.patchAlert(new Patch('add', '/comments', this.alertCommentsWrapper.map(alertsWrapper => alertsWrapper.alertComment)));
+ let commentRequest = new CommentAddRemoveRequest();
+ commentRequest.guid = this.alertSource.guid;
+ commentRequest.comment = this.alertCommentsWrapper[index].alertComment.comment;
+ commentRequest.username = this.alertCommentsWrapper[index].alertComment.username;
+ commentRequest.timestamp = this.alertCommentsWrapper[index].alertComment.timestamp;
+ commentRequest.sensorType = this.alertSourceType;
+ this.updateService.removeComment(commentRequest).subscribe( () => {
+ this.getData(true);
+ });
}
});
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-alerts/src/app/model/comment-add-remove-request.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/model/comment-add-remove-request.ts b/metron-interface/metron-alerts/src/app/model/comment-add-remove-request.ts
new file mode 100644
index 0000000..35f5d86
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/model/comment-add-remove-request.ts
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export class CommentAddRemoveRequest {
+ guid: string;
+ comment: string;
+ username: string;
+ sensorType: string;
+ timestamp: number;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-alerts/src/app/service/update.service.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-alerts/src/app/service/update.service.ts b/metron-interface/metron-alerts/src/app/service/update.service.ts
index 24b55f0..42a4944 100644
--- a/metron-interface/metron-alerts/src/app/service/update.service.ts
+++ b/metron-interface/metron-alerts/src/app/service/update.service.ts
@@ -31,6 +31,7 @@ import {Utils} from '../utils/utils';
import {Patch} from '../model/patch';
import {META_ALERTS_INDEX, META_ALERTS_SENSOR_TYPE} from '../utils/constants';
import { GlobalConfigService } from './global-config.service';
+import {CommentAddRemoveRequest} from "../model/comment-add-remove-request";
@Injectable()
export class UpdateService {
@@ -40,6 +41,8 @@ export class UpdateService {
alertChangedSource = new Subject<PatchRequest>();
alertChanged$ = this.alertChangedSource.asObservable();
sourceType = 'source:type';
+ alertCommentChangedSource = new Subject<CommentAddRemoveRequest>();
+ alertCommentChanged$ = this.alertCommentChangedSource.asObservable();
constructor(private http: Http, private globalConfigService: GlobalConfigService) {
this.globalConfigService.get().subscribe((config: {}) => {
@@ -47,6 +50,30 @@ export class UpdateService {
});
}
+ public addComment(commentRequest: CommentAddRemoveRequest, fireChangeListener = true): Observable<{}> {
+ let url = '/api/v1/update/add/comment';
+ return this.http.post(url, commentRequest, new RequestOptions({headers: new Headers(this.defaultHeaders)}))
+ .catch(HttpUtil.handleError)
+ .map(result => {
+ if (fireChangeListener) {
+ this.alertCommentChangedSource.next(commentRequest);
+ }
+ return result;
+ });
+ }
+
+ public removeComment(commentRequest: CommentAddRemoveRequest, fireChangeListener = true): Observable<{}> {
+ let url = '/api/v1/update/remove/comment';
+ return this.http.post(url, commentRequest, new RequestOptions({headers: new Headers(this.defaultHeaders)}))
+ .catch(HttpUtil.handleError)
+ .map(result => {
+ if (fireChangeListener) {
+ this.alertCommentChangedSource.next(commentRequest);
+ }
+ return result;
+ });
+ }
+
public patch(patchRequest: PatchRequest, fireChangeListener = true): Observable<{}> {
let url = '/api/v1/update/patch';
return this.http.patch(url, patchRequest, new RequestOptions({headers: new Headers(this.defaultHeaders)}))
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java
index 56b0b7b..609442b 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UpdateController.java
@@ -20,6 +20,7 @@ package org.apache.metron.rest.controller;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import io.swagger.annotations.ApiResponse;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
import org.apache.metron.indexing.dao.update.ReplaceRequest;
@@ -67,4 +68,26 @@ public class UpdateController {
service.replace(request);
return new ResponseEntity<>(HttpStatus.OK);
}
+
+ @ApiOperation(value = "Add a comment to an alert")
+ @ApiResponse(message = "Nothing", code = 200)
+ @RequestMapping(value = "/add/comment", method = RequestMethod.POST)
+ ResponseEntity<Void> addCommentToAlert(
+ @RequestBody @ApiParam(name = "request", value = "Comment add request", required = true) final
+ CommentAddRemoveRequest request
+ ) throws RestException {
+ service.addComment(request);
+ return new ResponseEntity<>(HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Remove a comment to an alert")
+ @ApiResponse(message = "Nothing", code = 200)
+ @RequestMapping(value = "/remove/comment", method = RequestMethod.POST)
+ ResponseEntity<Void> removeCommentFromAlert(
+ @RequestBody @ApiParam(name = "request", value = "Comment remove request", required = true) final
+ CommentAddRemoveRequest request
+ ) throws RestException {
+ service.removeComment(request);
+ return new ResponseEntity<>(HttpStatus.OK);
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java
index 4cdf4b3..bd59f39 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/UpdateService.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.rest.service;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
import org.apache.metron.indexing.dao.update.ReplaceRequest;
@@ -26,4 +27,6 @@ public interface UpdateService {
void patch(PatchRequest request) throws RestException, OriginalNotFoundException;
void replace(ReplaceRequest request) throws RestException;
+ void addComment(CommentAddRemoveRequest request) throws RestException;
+ void removeComment(CommentAddRemoveRequest request) throws RestException;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java
index 6a42248..49490fd 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/UpdateServiceImpl.java
@@ -18,6 +18,7 @@
package org.apache.metron.rest.service.impl;
import org.apache.metron.indexing.dao.IndexDao;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
import org.apache.metron.indexing.dao.update.ReplaceRequest;
@@ -59,4 +60,22 @@ public class UpdateServiceImpl implements UpdateService {
throw new RestException(e.getMessage(), e);
}
}
+
+ @Override
+ public void addComment(CommentAddRemoveRequest request) throws RestException {
+ try {
+ dao.addCommentToAlert(request);
+ } catch (Exception e) {
+ throw new RestException(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public void removeComment(CommentAddRemoveRequest request) throws RestException {
+ try {
+ dao.removeCommentFromAlert(request);
+ } catch (Exception e) {
+ throw new RestException(e.getMessage(), e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
index e437325..6b8d5d3 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
@@ -17,15 +17,29 @@
*/
package org.apache.metron.rest.controller;
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
import com.google.common.collect.ImmutableMap;
+import java.util.NavigableMap;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.curator.framework.CuratorFramework;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.metron.hbase.mock.MockHTable;
+import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.hbase.mock.MockHBaseTableProvider;
+import org.apache.metron.hbase.mock.MockHTable;
import org.apache.metron.indexing.dao.HBaseDao;
import org.apache.metron.indexing.dao.SearchIntegrationTest;
+import org.apache.metron.indexing.dao.search.AlertComment;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.rest.service.UpdateService;
import org.junit.Assert;
import org.junit.Before;
@@ -37,28 +51,17 @@ import org.springframework.http.MediaType;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.MvcResult;
import org.springframework.test.web.servlet.ResultActions;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.context.WebApplicationContext;
-import java.util.NavigableMap;
-
-import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
-import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
-import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
-import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
-
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@ActiveProfiles(TEST_PROFILE)
public class UpdateControllerIntegrationTest extends DaoControllerTest {
@Autowired
- private UpdateService searchService;
+ private UpdateService updateService;
@Autowired
public CuratorFramework client;
@@ -115,6 +118,30 @@ public class UpdateControllerIntegrationTest extends DaoControllerTest {
@Multiline
public static String replace;
+ /**
+ {
+ "guid" : "bro_2",
+ "sensorType" : "bro",
+ "comment": "test_comment",
+ "username" : "test_username",
+ "timestamp":0
+ }
+ */
+ @Multiline
+ public static String addComment;
+
+ /**
+ {
+ "guid" : "bro_2",
+ "sensorType" : "bro",
+ "comment": "test_comment",
+ "username" : "test_username",
+ "timestamp":0
+ }
+ */
+ @Multiline
+ public static String removeComment;
+
@Before
public void setup() throws Exception {
this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
@@ -191,4 +218,41 @@ public class UpdateControllerIntegrationTest extends DaoControllerTest {
}
}
+ @Test
+ public void shouldAddComment() throws Exception {
+ CommentAddRemoveRequest commentAddRemoveRequest = new CommentAddRemoveRequest();
+ commentAddRemoveRequest.setGuid("bro_1");
+ commentAddRemoveRequest.setSensorType("bro");
+ commentAddRemoveRequest.setComment("test_comment");
+ commentAddRemoveRequest.setUsername("test_username");
+ commentAddRemoveRequest.setTimestamp(0L);
+
+ updateService.addComment(commentAddRemoveRequest);
+
+ ResultActions result = this.mockMvc.perform(
+ post(updateUrl + "/add/comment")
+ .with(httpBasic(user, password)).with(csrf())
+ .contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))
+ .content(addComment));
+ result.andExpect(status().isOk());
+ }
+
+ @Test
+ public void shouldRemoveComment() throws Exception {
+ CommentAddRemoveRequest commentAddRemoveRequest = new CommentAddRemoveRequest();
+ commentAddRemoveRequest.setGuid("bro_1");
+ commentAddRemoveRequest.setSensorType("bro");
+ commentAddRemoveRequest.setComment("test_comment");
+ commentAddRemoveRequest.setUsername("test_username");
+ commentAddRemoveRequest.setTimestamp(0L);
+
+ updateService.removeComment(commentAddRemoveRequest);
+
+ ResultActions result = this.mockMvc.perform(
+ post(updateUrl + "/remove/comment")
+ .with(httpBasic(user, password)).with(csrf())
+ .contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))
+ .content(removeComment));
+ result.andExpect(status().isOk());
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
index 246de6a..eae0a39 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
@@ -33,6 +33,7 @@ import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
@@ -150,6 +151,16 @@ public class ElasticsearchDao implements IndexDao {
}
@Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ updateDao.addCommentToAlert(request);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ updateDao.removeCommentFromAlert(request);
+ }
+
+ @Override
public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException {
return this.columnMetadataDao.getColumnMetadata(indices);
}
@@ -159,6 +170,16 @@ public class ElasticsearchDao implements IndexDao {
return retrieveLatestDao.getLatestResult(request);
}
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ this.updateDao.addCommentToAlert(request, latest);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ this.updateDao.removeCommentFromAlert(request, latest);
+ }
+
protected Optional<String> getIndexName(String guid, String sensorType) {
return updateDao.getIndexName(guid, sensorType);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
index faec939..ab6c40c 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
@@ -41,6 +41,8 @@ import org.apache.metron.indexing.dao.search.InvalidCreateException;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.search.SearchResult;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
@@ -216,6 +218,26 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
}
@Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ indexDao.addCommentToAlert(request);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ indexDao.removeCommentFromAlert(request);
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ indexDao.addCommentToAlert(request, latest);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ indexDao.removeCommentFromAlert(request, latest);
+ }
+
+ @Override
public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
Optional<Long> timestamp)
throws OriginalNotFoundException, IOException {
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
index 6c709a6..d3bdcbb 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertUpdateDao.java
@@ -44,6 +44,7 @@ import org.apache.metron.indexing.dao.metaalert.lucene.AbstractLuceneMetaAlertUp
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.InvalidCreateException;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.elasticsearch.index.query.InnerHitBuilder;
import org.elasticsearch.index.query.QueryBuilder;
@@ -185,6 +186,28 @@ public class ElasticsearchMetaAlertUpdateDao extends AbstractLuceneMetaAlertUpda
}
}
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ getUpdateDao().addCommentToAlert(request);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ getUpdateDao().removeCommentFromAlert(request);
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest)
+ throws IOException {
+ getUpdateDao().addCommentToAlert(request, latest);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
+ throws IOException {
+ getUpdateDao().removeCommentFromAlert(request, latest);
+ }
+
/**
* Given an alert GUID, retrieve all associated meta alerts.
* @param alertGuid The GUID of the child alert
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java
index c4d7412..f2b08d2 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchUpdateDao.java
@@ -17,14 +17,21 @@
*/
package org.apache.metron.elasticsearch.dao;
+import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD;
+
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.stream.Collectors;
import org.apache.metron.elasticsearch.utils.ElasticsearchUtils;
import org.apache.metron.indexing.dao.AccessConfig;
+import org.apache.metron.indexing.dao.search.AlertComment;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.UpdateDao;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
@@ -103,6 +110,69 @@ public class ElasticsearchUpdateDao implements UpdateDao {
}
}
+ @Override
+ @SuppressWarnings("unchecked")
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ Document latest = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType());
+ addCommentToAlert(request, latest);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ if (latest == null) {
+ return;
+ }
+ List<Map<String, Object>> commentsField = (List<Map<String, Object>>) latest.getDocument()
+ .getOrDefault(COMMENTS_FIELD, new ArrayList<>());
+ List<Map<String, Object>> originalComments = new ArrayList<>(commentsField);
+
+ originalComments.add(
+ new AlertComment(request.getComment(), request.getUsername(), request.getTimestamp())
+ .asMap());
+
+ Document newVersion = new Document(latest);
+ newVersion.getDocument().put(COMMENTS_FIELD, originalComments);
+ update(newVersion, Optional.empty());
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ Document latest = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType());
+ removeCommentFromAlert(request, latest);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ if (latest == null) {
+ return;
+ }
+ List<Map<String, Object>> commentsField = (List<Map<String, Object>>) latest.getDocument()
+ .getOrDefault(COMMENTS_FIELD, new ArrayList<>());
+ List<Map<String, Object>> originalComments = new ArrayList<>(commentsField);
+
+ List<AlertComment> alertComments = new ArrayList<>();
+ for (Map<String, Object> commentRaw : originalComments) {
+ alertComments.add(new AlertComment(commentRaw));
+ }
+
+ alertComments.remove(
+ new AlertComment(request.getComment(), request.getUsername(), request.getTimestamp()));
+ List<Map<String, Object>> commentsFinal = alertComments.stream().map(AlertComment::asMap)
+ .collect(Collectors.toList());
+ Document newVersion = new Document(latest);
+ if (commentsFinal.size() > 0) {
+ newVersion.getDocument().put(COMMENTS_FIELD, commentsFinal);
+ update(newVersion, Optional.empty());
+ } else {
+ newVersion.getDocument().remove(COMMENTS_FIELD);
+ }
+
+ update(newVersion, Optional.empty());
+ }
+
protected String getIndexName(Document update, Optional<String> index, String indexPostFix) {
return index.orElse(getIndexName(update.getGuid(), update.getSensorType())
.orElse(ElasticsearchUtils.getIndexName(update.getSensorType(), indexPostFix, null))
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
index 25799ad..a3a5f16 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java
@@ -35,6 +35,7 @@ import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.InvalidCreateException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.junit.Test;
@@ -81,6 +82,22 @@ public class ElasticsearchMetaAlertDaoTest {
public Map<String, FieldType> getColumnMetadata(List<String> indices) {
return null;
}
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ }
};
ElasticsearchMetaAlertDao metaAlertDao = new ElasticsearchMetaAlertDao();
metaAlertDao.init(dao);
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
index 97993ff..c5c0bc1 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
@@ -85,8 +85,9 @@ public class ElasticsearchUpdateIntegrationTest extends UpdateIntegrationTest {
globalConfig.put(HBaseDao.HBASE_CF, CF);
accessConfig.setGlobalConfigSupplier(() -> globalConfig);
- dao = new MultiIndexDao(hbaseDao, createDao());
+ MultiIndexDao dao = new MultiIndexDao(hbaseDao, createDao());
dao.init(accessConfig);
+ setDao(dao);
}
@After
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
index ebb9907..f22372e 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
@@ -28,8 +28,7 @@ import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Optional;
-
-import com.google.common.hash.Hasher;
+import java.util.stream.Collectors;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTableInterface;
@@ -38,6 +37,7 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.common.utils.KeyUtil;
+import org.apache.metron.indexing.dao.search.AlertComment;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.GroupRequest;
@@ -45,6 +45,7 @@ import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
/**
@@ -210,7 +211,21 @@ public class HBaseDao implements IndexDao {
if(entry.getValue()!= null) {
Map<String, Object> json = JSONUtils.INSTANCE.load(new String(entry.getValue()),
JSONUtils.MAP_SUPPLIER);
+
+ // Make sure comments are in the proper format
+ @SuppressWarnings("unchecked")
+ List<Map<String, Object>> commentsMap = (List<Map<String, Object>>) json.get(COMMENTS_FIELD);
try {
+ if (commentsMap != null) {
+ List<AlertComment> comments = new ArrayList<>();
+ for (Map<String, Object> commentMap : commentsMap) {
+ comments.add(new AlertComment(commentMap));
+ }
+ if (comments.size() > 0) {
+ json.put(COMMENTS_FIELD,
+ comments.stream().map(AlertComment::asMap).collect(Collectors.toList()));
+ }
+ }
Key k = Key.fromBytes(result.getRow());
return new Document(json, k.getGuid(), k.getSensorType(), ts);
} catch (IOException e) {
@@ -262,4 +277,78 @@ public class HBaseDao implements IndexDao {
public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException {
return null;
}
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ Document latest = getLatest(request.getGuid(), request.getSensorType());
+ addCommentToAlert(request, latest);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ if (latest == null || latest.getDocument() == null) {
+ throw new IOException("Unable to add comment to document that doesn't exist");
+ }
+
+ List<Map<String, Object>> comments = (List<Map<String, Object>>) latest.getDocument()
+ .getOrDefault(COMMENTS_FIELD, new ArrayList<>());
+ List<Map<String, Object>> originalComments = new ArrayList<>(comments);
+
+ // Convert all comments back to raw JSON before updating.
+ List<Map<String, Object>> commentsMap = new ArrayList<>();
+ for (Map<String, Object> comment : originalComments) {
+ commentsMap.add(new AlertComment(comment).asMap());
+ }
+ commentsMap.add(new AlertComment(
+ request.getComment(),
+ request.getUsername(),
+ request.getTimestamp())
+ .asMap());
+
+ Document newVersion = new Document(latest);
+ newVersion.getDocument().put(COMMENTS_FIELD, commentsMap);
+ update(newVersion, Optional.empty());
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void removeCommentFromAlert(CommentAddRemoveRequest request)
+ throws IOException {
+ Document latest = getLatest(request.getGuid(), request.getSensorType());
+ removeCommentFromAlert(request, latest);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest)
+ throws IOException {
+ if (latest == null || latest.getDocument() == null) {
+ throw new IOException("Unable to remove comment document that doesn't exist");
+ }
+ List<Map<String, Object>> commentMap = (List<Map<String, Object>>) latest.getDocument().get(COMMENTS_FIELD);
+ // Can't remove anything if there's nothing there
+ if (commentMap == null) {
+ return;
+ }
+ List<Map<String, Object>> originalComments = new ArrayList<>(commentMap);
+ List<AlertComment> comments = new ArrayList<>();
+ for (Map<String, Object> commentStr : originalComments) {
+ comments.add(new AlertComment(commentStr));
+ }
+
+ comments.remove(new AlertComment(request.getComment(), request.getUsername(), request.getTimestamp()));
+ Document newVersion = new Document(latest);
+ if (comments.size() > 0) {
+ List<Map<String, Object>> commentsAsMap = comments.stream().map(AlertComment::asMap)
+ .collect(Collectors.toList());
+ newVersion.getDocument().put(COMMENTS_FIELD, commentsAsMap);
+ update(newVersion, Optional.empty());
+ } else {
+ newVersion.getDocument().remove(COMMENTS_FIELD);
+ }
+
+ update(newVersion, Optional.empty());
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
index 4187428..11b2ff0 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
@@ -26,6 +26,8 @@ import org.apache.metron.indexing.dao.update.UpdateDao;
*/
public interface IndexDao extends UpdateDao, SearchDao, RetrieveLatestDao, ColumnMetadataDao {
+ String COMMENTS_FIELD = "comments";
+
/**
* Initialize the DAO with the AccessConfig object.
* @param config The config to use for initialization
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
index dad08d6..420c775 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MultiIndexDao.java
@@ -25,6 +25,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -36,6 +37,7 @@ import org.apache.metron.indexing.dao.search.GroupResponse;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
public class MultiIndexDao implements IndexDao {
@@ -98,6 +100,51 @@ public class MultiIndexDao implements IndexDao {
return null;
}
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) throws IOException {
+ Document latest = getLatest(request.getGuid(), request.getSensorType());
+ addCommentToAlert(request, latest);
+ }
+
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ List<String> exceptions =
+ indices.parallelStream().map(dao -> {
+ try {
+ dao.addCommentToAlert(request, latest);
+ return null;
+ } catch (Throwable e) {
+ return dao.getClass() + ": " + e.getMessage() + "\n" + ExceptionUtils.getStackTrace(e);
+ }
+ }).filter(Objects::nonNull).collect(Collectors.toList());
+ if (exceptions.size() > 0) {
+ throw new IOException(Joiner.on("\n").join(exceptions));
+ }
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException {
+ Document latest = getLatest(request.getGuid(), request.getSensorType());
+ removeCommentFromAlert(request, latest);
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException {
+ List<String> exceptions =
+ indices.parallelStream().map(dao -> {
+ try {
+ dao.removeCommentFromAlert(request, latest);
+ return null;
+ } catch (Throwable e) {
+ return dao.getClass() + ": " + e.getMessage() + "\n" + ExceptionUtils.getStackTrace(e);
+ }
+ }).filter(Objects::nonNull).collect(Collectors.toList());
+ if (exceptions.size() > 0) {
+ throw new IOException(Joiner.on("\n").join(exceptions));
+ }
+ }
+
private static class DocumentContainer {
private Optional<Document> d = Optional.empty();
private Optional<Throwable> t = Optional.empty();
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/AlertComment.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/AlertComment.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/AlertComment.java
new file mode 100644
index 0000000..04aac60
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/AlertComment.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao.search;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+
+public class AlertComment {
+
+ private static final String COMMENT_FIELD = "comment";
+ private static final String COMMENT_USERNAME_FIELD = "username";
+ private static final String COMMENT_TIMESTAMP_FIELD = "timestamp";
+ private String comment;
+ private String username;
+ private long timestamp;
+
+ private JSONParser parser = new JSONParser();
+
+ public AlertComment(String comment, String username, long timestamp) {
+ this.comment = comment;
+ this.username = username;
+ this.timestamp = timestamp;
+ }
+
+ public AlertComment(String json) throws ParseException {
+ JSONObject parsed = (JSONObject) parser.parse(json);
+ this.comment = (String) parsed.get(COMMENT_FIELD);
+ this.username = (String) parsed.get(COMMENT_USERNAME_FIELD);
+ this.timestamp = (long) parsed.get(COMMENT_TIMESTAMP_FIELD);
+ }
+
+ public AlertComment(Map<String, Object> comment) {
+ this.comment = (String) comment.get(COMMENT_FIELD);
+ this.username = (String) comment.get(COMMENT_USERNAME_FIELD);
+ this.timestamp = (long) comment.get(COMMENT_TIMESTAMP_FIELD);
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public String getUsername() {
+ return username;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ @SuppressWarnings("unchecked")
+ public String asJson() {
+ return asJSONObject().toJSONString();
+ }
+
+ @SuppressWarnings("unchecked")
+ public Map<String, Object> asMap() {
+ Map<String, Object> map = new HashMap<>();
+ map.put(COMMENT_FIELD, comment);
+ map.put(COMMENT_USERNAME_FIELD, username);
+ map.put(COMMENT_TIMESTAMP_FIELD, timestamp);
+ return map;
+ }
+
+ @SuppressWarnings("unchecked")
+ public JSONObject asJSONObject() {
+ JSONObject json = new JSONObject();
+ json.put(COMMENT_FIELD, comment);
+ json.put(COMMENT_USERNAME_FIELD, username);
+ json.put(COMMENT_TIMESTAMP_FIELD, timestamp);
+ return json;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ AlertComment that = (AlertComment) o;
+
+ if (getTimestamp() != that.getTimestamp()) {
+ return false;
+ }
+ if (getComment() != null ? !getComment().equals(that.getComment())
+ : that.getComment() != null) {
+ return false;
+ }
+ return getUsername() != null ? getUsername().equals(that.getUsername())
+ : that.getUsername() == null;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = getComment() != null ? getComment().hashCode() : 0;
+ result = 31 * result + (getUsername() != null ? getUsername().hashCode() : 0);
+ result = 31 * result + (int) (getTimestamp() ^ (getTimestamp() >>> 32));
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "AlertComment{" +
+ "comment='" + comment + '\'' +
+ ", username='" + username + '\'' +
+ ", timestamp=" + timestamp +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/CommentAddRemoveRequest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/CommentAddRemoveRequest.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/CommentAddRemoveRequest.java
new file mode 100644
index 0000000..8e8bde7
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/CommentAddRemoveRequest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao.update;
+
+public class CommentAddRemoveRequest {
+ private String guid;
+ private String sensorType;
+ private String comment;
+ private String username;
+ private long timestamp;
+
+ public String getGuid() {
+ return guid;
+ }
+
+ public void setGuid(String guid) {
+ this.guid = guid;
+ }
+
+ public String getSensorType() {
+ return sensorType;
+ }
+
+ public void setSensorType(String sensorType) {
+ this.sensorType = sensorType;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ public String getUsername() {
+ return username;
+ }
+
+ public void setUsername(String username) {
+ this.username = username;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ @Override
+ public String toString() {
+ return "CommentAddRemoveRequest{" +
+ "guid='" + guid + '\'' +
+ ", sensorType='" + sensorType + '\'' +
+ ", comment='" + comment + '\'' +
+ ", username='" + username + '\'' +
+ ", timestamp=" + timestamp +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java
index 6f2f779..3686b19 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java
@@ -18,10 +18,10 @@
package org.apache.metron.indexing.dao.update;
-import org.apache.metron.common.utils.JSONUtils;
-
import java.io.IOException;
+import java.util.HashMap;
import java.util.Map;
+import org.apache.metron.common.utils.JSONUtils;
public class Document {
Long timestamp;
@@ -36,7 +36,6 @@ public class Document {
setSensorType(sensorType);
}
-
public Document(String document, String guid, String sensorType, Long timestamp) throws IOException {
this(convertDoc(document), guid, sensorType, timestamp);
}
@@ -45,6 +44,15 @@ public class Document {
this( document, guid, sensorType, null);
}
+ /**
+ * Copy constructor
+ * @param other The document to be copied.
+ */
+ public Document(Document other) {
+ this(new HashMap<>(other.getDocument()), other.getGuid(), other.getSensorType(),
+ other.getTimestamp());
+ }
+
private static Map<String, Object> convertDoc(String document) throws IOException {
return JSONUtils.INSTANCE.load(document, JSONUtils.MAP_SUPPLIER);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/PatchUtil.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/PatchUtil.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/PatchUtil.java
deleted file mode 100644
index 5a4ef27..0000000
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/PatchUtil.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.indexing.dao.update;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Optional;
-import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.indexing.dao.RetrieveLatestDao;
-
-public class PatchUtil {
-
- public static Document getPatchedDocument(
- RetrieveLatestDao retrieveLatestDao,
- PatchRequest request
- , Optional<Long> timestamp
- ) throws OriginalNotFoundException, IOException {
- Map<String, Object> latest = request.getSource();
- if (latest == null) {
- Document latestDoc = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType());
- if (latestDoc != null && latestDoc.getDocument() != null) {
- latest = latestDoc.getDocument();
- } else {
- throw new OriginalNotFoundException(
- "Unable to patch an document that doesn't exist and isn't specified.");
- }
- }
- Map<String, Object> updated = JSONUtils.INSTANCE.applyPatch(request.getPatch(), latest);
- return new Document(updated
- , request.getGuid()
- , request.getSensorType()
- , timestamp.orElse(System.currentTimeMillis()));
- }
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java
index 6f136ea..b5f38e4 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/UpdateDao.java
@@ -18,8 +18,10 @@
package org.apache.metron.indexing.dao.update;
import java.io.IOException;
+import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
+import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.RetrieveLatestDao;
public interface UpdateDao {
@@ -43,6 +45,15 @@ public interface UpdateDao {
*/
void batchUpdate(Map<Document, Optional<String>> updates) throws IOException;
+ void addCommentToAlert(CommentAddRemoveRequest request) throws IOException;
+
+ void removeCommentFromAlert(CommentAddRemoveRequest request) throws IOException;
+
+ void addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException;
+
+ void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) throws IOException;
+
+
/**
* Update a document in an index given a JSON Patch (see RFC 6902 at
* https://tools.ietf.org/html/rfc6902)
@@ -54,10 +65,30 @@ public interface UpdateDao {
default void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request
, Optional<Long> timestamp
) throws OriginalNotFoundException, IOException {
- Document d = PatchUtil.getPatchedDocument(retrieveLatestDao, request, timestamp);
+ Document d = getPatchedDocument(retrieveLatestDao, request, timestamp);
update(d, Optional.ofNullable(request.getIndex()));
}
+ default Document getPatchedDocument(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
+ Optional<Long> timestamp
+ ) throws OriginalNotFoundException, IOException {
+ Map<String, Object> latest = request.getSource();
+ if (latest == null) {
+ Document latestDoc = retrieveLatestDao.getLatest(request.getGuid(), request.getSensorType());
+ if (latestDoc != null && latestDoc.getDocument() != null) {
+ latest = latestDoc.getDocument();
+ } else {
+ throw new OriginalNotFoundException(
+ "Unable to patch an document that doesn't exist and isn't specified.");
+ }
+ }
+
+ Map<String, Object> updated = JSONUtils.INSTANCE.applyPatch(request.getPatch(), latest);
+ return new Document(updated,
+ request.getGuid(),
+ request.getSensorType(),
+ timestamp.orElse(System.currentTimeMillis()));
+ }
/**
* Replace a document in an index.
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
index d6e1521..e306567 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
@@ -35,6 +35,7 @@ import org.apache.metron.indexing.dao.search.SearchResponse;
import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.indexing.dao.search.SortField;
import org.apache.metron.indexing.dao.search.SortOrder;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import java.io.IOException;
@@ -291,6 +292,22 @@ public class InMemoryDao implements IndexDao {
return indexColumnMetadata;
}
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ }
+
public static void setColumnMetadata(Map<String, Map<String, FieldType>> columnMetadata) {
Map<String, Map<String, FieldType>> columnMetadataMap = new HashMap<>();
for (Map.Entry<String, Map<String, FieldType>> e: columnMetadata.entrySet()) {
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
index 803d320..9e95ee9 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryMetaAlertDao.java
@@ -47,6 +47,7 @@ import org.apache.metron.indexing.dao.search.InvalidSearchException;
import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
import org.apache.metron.indexing.dao.search.SearchResult;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
@@ -127,6 +128,22 @@ public class InMemoryMetaAlertDao implements MetaAlertDao {
}
@Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ }
+
+ @Override
public Optional<Map<String, Object>> getLatestResult(GetRequest request) throws IOException {
return indexDao.getLatestResult(request);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
index eebf0bb..1e35523 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/UpdateIntegrationTest.java
@@ -14,30 +14,61 @@
*/
package org.apache.metron.indexing.dao;
+import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD;
+
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Optional;
+import java.util.stream.Collectors;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.commons.collections.MapUtils;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.metron.common.Constants;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.hbase.mock.MockHTable;
+import org.apache.metron.indexing.dao.search.AlertComment;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
+import org.apache.metron.indexing.dao.update.PatchRequest;
import org.apache.metron.indexing.dao.update.ReplaceRequest;
import org.junit.Assert;
import org.junit.Test;
public abstract class UpdateIntegrationTest {
+ /**
+ * {
+ * "comment":"New Comment",
+ * "username":"test_user",
+ * "timestamp":1526401584951
+ * }
+ */
+ @Multiline
+ protected String commentOne;
+
+ /**
+ * {
+ * "comment":"New Comment 2",
+ * "username":"test_user_2",
+ * "timestamp":1526401584952
+ * }
+ */
+ @Multiline
+ protected String commentTwo;
+
private static final int MAX_RETRIES = 10;
private static final int SLEEP_MS = 500;
protected static final String SENSOR_NAME = "test";
private static final String CF = "p";
- protected static MultiIndexDao dao;
+ private MultiIndexDao dao;
@Test
public void test() throws Exception {
@@ -68,7 +99,7 @@ public abstract class UpdateIntegrationTest {
put("new-field", "metron");
}};
String guid = "" + message0.get(Constants.GUID);
- dao.replace(new ReplaceRequest(){{
+ getDao().replace(new ReplaceRequest(){{
setReplacement(message0);
setGuid(guid);
setSensorType(SENSOR_NAME);
@@ -76,8 +107,7 @@ public abstract class UpdateIntegrationTest {
}}, Optional.empty());
Assert.assertEquals(1, getMockHTable().size());
- Document doc = dao.getLatest(guid, SENSOR_NAME);
- Assert.assertEquals(message0, doc.getDocument());
+ findUpdatedDoc(message0, guid, SENSOR_NAME);
{
//ensure hbase is up to date
Get g = new Get(HBaseDao.Key.toBytes(new HBaseDao.Key(guid, SENSOR_NAME)));
@@ -99,7 +129,7 @@ public abstract class UpdateIntegrationTest {
.filter(d -> message0.get("new-field").equals(d.get("new-field")))
.count();
}
- Assert.assertNotEquals("Elasticsearch is not updated!", cnt, 0);
+ Assert.assertNotEquals("Data store is not updated!", cnt, 0);
}
}
//modify the same message and modify the new field
@@ -108,15 +138,16 @@ public abstract class UpdateIntegrationTest {
put("new-field", "metron2");
}};
String guid = "" + message0.get(Constants.GUID);
- dao.replace(new ReplaceRequest(){{
+ getDao().replace(new ReplaceRequest(){{
setReplacement(message0);
setGuid(guid);
setSensorType(SENSOR_NAME);
setIndex(getIndexName());
}}, Optional.empty());
Assert.assertEquals(1, getMockHTable().size());
- Document doc = dao.getLatest(guid, SENSOR_NAME);
+ Document doc = getDao().getLatest(guid, SENSOR_NAME);
Assert.assertEquals(message0, doc.getDocument());
+ findUpdatedDoc(message0, guid, SENSOR_NAME);
{
//ensure hbase is up to date
Get g = new Get(HBaseDao.Key.toBytes(new HBaseDao.Key(guid, SENSOR_NAME)));
@@ -141,11 +172,133 @@ public abstract class UpdateIntegrationTest {
.count();
}
- Assert.assertNotEquals("Index is not updated!", cnt, 0);
+ Assert.assertNotEquals("Data store is not updated!", cnt, 0);
}
}
}
+ @Test
+ public void testAddCommentAndPatch() throws Exception {
+ Map<String, Object> fields = new HashMap<>();
+ fields.put("guid", "add_comment");
+ fields.put("source.type", SENSOR_NAME);
+
+ Document document = new Document(fields, "add_comment", SENSOR_NAME, 1526306463050L);
+ getDao().update(document, Optional.of(SENSOR_NAME));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ addAlertComment("add_comment", "New Comment", "test_user", 1526306463050L);
+ // Ensure we have the first comment
+ ArrayList<AlertComment> comments = new ArrayList<>();
+ comments.add(new AlertComment("New Comment", "test_user", 1526306463050L));
+ document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
+ Collectors.toList()));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ List<Map<String, Object>> patchList = new ArrayList<>();
+ Map<String, Object> patch = new HashMap<>();
+ patch.put("op", "add");
+ patch.put("path", "/project");
+ patch.put("value", "metron");
+ patchList.add(patch);
+
+ PatchRequest pr = new PatchRequest();
+ pr.setGuid("add_comment");
+ pr.setIndex(SENSOR_NAME);
+ pr.setSensorType(SENSOR_NAME);
+ pr.setPatch(patchList);
+ getDao().patch(getDao(), pr, Optional.of(new Date().getTime()));
+
+ document.getDocument().put("project", "metron");
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testRemoveComments() throws Exception {
+ Map<String, Object> fields = new HashMap<>();
+ fields.put("guid", "add_comment");
+ fields.put("source.type", SENSOR_NAME);
+
+ Document document = new Document(fields, "add_comment", SENSOR_NAME, 1526401584951L);
+ getDao().update(document, Optional.of(SENSOR_NAME));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ addAlertComment("add_comment", "New Comment", "test_user", 1526401584951L);
+ // Ensure we have the first comment
+ ArrayList<AlertComment> comments = new ArrayList<>();
+ comments.add(new AlertComment("New Comment", "test_user", 1526401584951L));
+ document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
+ Collectors.toList()));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ addAlertComment("add_comment", "New Comment 2", "test_user_2", 1526401584952L);
+ // Ensure we have the second comment
+ comments.add(new AlertComment("New Comment 2", "test_user_2", 1526401584952L));
+ document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
+ Collectors.toList()));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ removeAlertComment("add_comment", "New Comment 2", "test_user_2", 1526401584952L);
+ // Ensure we only have the first comments
+ comments = new ArrayList<>();
+ comments.add(new AlertComment(commentOne));
+ document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
+ Collectors.toList()));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ removeAlertComment("add_comment", "New Comment", "test_user", 1526401584951L);
+ // Ensure we have no comments
+ document.getDocument().remove(COMMENTS_FIELD);
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+ }
+
+ protected void addAlertComment(String guid, String comment, String username, long timestamp)
+ throws IOException {
+ CommentAddRemoveRequest request = buildAlertRequest(guid, comment, username, timestamp);
+ getDao().addCommentToAlert(request);
+ }
+
+ protected void removeAlertComment(String guid, String comment, String username, long timestamp)
+ throws IOException {
+ CommentAddRemoveRequest request = buildAlertRequest(guid, comment, username, timestamp);
+ getDao().removeCommentFromAlert(request);
+ }
+
+ private CommentAddRemoveRequest buildAlertRequest(String guid, String comment, String username,
+ long timestamp) {
+ CommentAddRemoveRequest request = new CommentAddRemoveRequest();
+ request.setGuid(guid);
+ request.setComment(comment);
+ request.setUsername(username);
+ request.setTimestamp(timestamp);
+ request.setSensorType(SENSOR_NAME);
+ return request;
+ }
+
+ protected void findUpdatedDoc(Map<String, Object> message0, String guid, String sensorType)
+ throws InterruptedException, IOException, OriginalNotFoundException {
+ for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) {
+ Document doc = getDao().getLatest(guid, sensorType);
+ if (doc != null && message0.equals(doc.getDocument())) {
+ return;
+ }
+ if (t == MAX_RETRIES -1) {
+ MapUtils.debugPrint(System.out, "Expected", message0);
+ MapUtils.debugPrint(System.out, "actual", doc.getDocument());
+ }
+ }
+ throw new OriginalNotFoundException("Count not find " + guid + " after " + MAX_RETRIES + " tries");
+ }
+
+ protected IndexDao getDao() {
+ return dao;
+ }
+
+ protected void setDao(MultiIndexDao dao) {
+ this.dao = dao;
+ }
+
protected abstract String getIndexName();
protected abstract MockHTable getMockHTable();
protected abstract void addTestData(String indexName, String sensorType, List<Map<String,Object>> docs) throws Exception;
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java
index 2d620d9..7028b75 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDaoTest.java
@@ -58,6 +58,7 @@ import org.apache.metron.indexing.dao.metaalert.MetaAlertStatus;
import org.apache.metron.indexing.dao.metaalert.MetaScores;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.InvalidSearchException;
+import org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.PatchRequest;
import org.json.simple.JSONArray;
@@ -146,6 +147,22 @@ public class AbstractLuceneMetaAlertUpdateDaoTest {
}
@Override
+ public void addCommentToAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request) {
+ }
+
+ @Override
+ public void addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
+ }
+
+ @Override
+ public void removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
+ }
+
+ @Override
public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
Optional<Long> timestamp) {
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java
index aa32aa0..da74d46 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HBaseDaoIntegrationTest.java
@@ -20,6 +20,7 @@ package org.apache.metron.indexing.integration;
import static org.apache.metron.indexing.dao.HBaseDao.HBASE_CF;
import static org.apache.metron.indexing.dao.HBaseDao.HBASE_TABLE;
+import static org.apache.metron.indexing.dao.IndexDao.COMMENTS_FIELD;
import java.io.IOException;
import java.util.ArrayList;
@@ -29,12 +30,14 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
-
-import org.apache.commons.codec.binary.Hex;
import org.apache.metron.hbase.mock.MockHBaseTableProvider;
+import org.apache.metron.hbase.mock.MockHTable;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.HBaseDao;
import org.apache.metron.indexing.dao.IndexDao;
+import org.apache.metron.indexing.dao.MultiIndexDao;
+import org.apache.metron.indexing.dao.UpdateIntegrationTest;
+import org.apache.metron.indexing.dao.search.AlertComment;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.junit.After;
@@ -42,7 +45,7 @@ import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-public class HBaseDaoIntegrationTest {
+public class HBaseDaoIntegrationTest extends UpdateIntegrationTest {
private static final String TABLE_NAME = "metron_update";
private static final String COLUMN_FAMILY = "cf";
@@ -150,6 +153,13 @@ public class HBaseDaoIntegrationTest {
Assert.assertFalse("Result size should be 12 but was greater", results.hasNext());
}
+ @Override
+ public void test() {
+ // The main test ensures a variety of things not implemented by HBase run alongside
+ // HBaseDao itself.
+ // Therefore, just don't do anything for this test.
+ }
+
protected List<Document> buildAlerts(int count) throws IOException {
List<Document> alerts = new ArrayList<>();
for (int i = 0; i < count; ++i) {
@@ -161,4 +171,67 @@ public class HBaseDaoIntegrationTest {
return alerts;
}
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testRemoveComments() throws Exception {
+ Map<String, Object> fields = new HashMap<>();
+ fields.put("guid", "add_comment");
+ fields.put("source.type", SENSOR_NAME);
+
+ Document document = new Document(fields, "add_comment", SENSOR_NAME, 1526401584951L);
+ hbaseDao.update(document, Optional.of(SENSOR_NAME));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ addAlertComment("add_comment", "New Comment", "test_user", 1526401584951L);
+ // Ensure we have the first comment
+ ArrayList<AlertComment> comments = new ArrayList<>();
+ comments.add(new AlertComment("New Comment", "test_user", 1526401584951L));
+ document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
+ Collectors.toList()));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ addAlertComment("add_comment", "New Comment 2", "test_user_2", 1526401584952L);
+ // Ensure we have the second comment
+ comments.add(new AlertComment("New Comment 2", "test_user_2", 1526401584952L));
+ document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
+ Collectors.toList()));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ removeAlertComment("add_comment", "New Comment 2", "test_user_2", 1526401584952L);
+ // Ensure we only have the first comments
+ comments = new ArrayList<>();
+ comments.add(new AlertComment(commentOne));
+ document.getDocument().put(COMMENTS_FIELD, comments.stream().map(AlertComment::asMap).collect(
+ Collectors.toList()));
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+
+ removeAlertComment("add_comment", "New Comment", "test_user", 1526401584951L);
+ // Ensure we have no comments
+ document.getDocument().remove(COMMENTS_FIELD);
+ findUpdatedDoc(document.getDocument(), "add_comment", SENSOR_NAME);
+ }
+
+ @Override
+ protected IndexDao getDao() {
+ return hbaseDao;
+ }
+
+ @Override
+ protected String getIndexName() {
+ return null;
+ }
+
+ @Override
+ protected MockHTable getMockHTable() {
+ return null;
+ }
+
+ @Override
+ protected void addTestData(String indexName, String sensorType, List<Map<String, Object>> docs) {
+ }
+
+ @Override
+ protected List<Map<String, Object>> getIndexedTestData(String indexName, String sensorType) {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
index ca69304..1326dfc 100644
--- a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
+++ b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml
@@ -677,6 +677,9 @@
<dynamicField name="*.reason" type="string" multiValued="false" docValues="true"/>
<dynamicField name="*.name" type="string" multiValued="false" docValues="true"/>
+ <!-- Comments field required for the UI -->
+ <field name="comments" type="string" indexed="true" stored="true" multiValued="true"/>
+
<!-- Metaalerts Field -->
<field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/>
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
index 82d0320..84855df 100644
--- a/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
+++ b/metron-platform/metron-solr/src/main/config/schema/snort/schema.xml
@@ -70,6 +70,9 @@
<dynamicField name="*.reason" type="string" multiValued="false" docValues="true"/>
<dynamicField name="*.name" type="string" multiValued="false" docValues="true"/>
+ <!-- Comments field required for the UI -->
+ <field name="comments" type="string" indexed="true" stored="true" multiValued="true"/>
+
<!-- Metaalerts Field -->
<field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/>
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
index fc8e641..5555a14 100644
--- a/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
+++ b/metron-platform/metron-solr/src/main/config/schema/yaf/schema.xml
@@ -76,6 +76,9 @@
<dynamicField name="*.reason" type="string" multiValued="false" docValues="true"/>
<dynamicField name="*.name" type="string" multiValued="false" docValues="true"/>
+ <!-- Comments field required for the UI -->
+ <field name="comments" type="string" indexed="true" stored="true" multiValued="true"/>
+
<!-- Metaalerts Field -->
<field name="metaalerts" type="string" multiValued="true" indexed="true" stored="true"/>