You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by sa...@apache.org on 2019/07/04 08:02:33 UTC
[metron] branch master updated: METRON-2150 [UI] User not able to
filter by multiple values of the same field on Alerts UI (tiborm via
sardell) closes apache/metron#1443
This is an automated email from the ASF dual-hosted git repository.
sardell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 891ebd5 METRON-2150 [UI] User not able to filter by multiple values of the same field on Alerts UI (tiborm via sardell) closes apache/metron#1443
891ebd5 is described below
commit 891ebd5ce4391bcf3260cff3db20f08d27ea21fa
Author: tiborm <ti...@gmail.com>
AuthorDate: Thu Jul 4 10:02:00 2019 +0200
METRON-2150 [UI] User not able to filter by multiple values of the same field on Alerts UI (tiborm via sardell) closes apache/metron#1443
---
.../alerts/alerts-list/alerts-list.component.ts | 4 +-
.../app/alerts/alerts-list/query-builder.spec.ts | 133 +++++++++++++++++++++
.../src/app/alerts/alerts-list/query-builder.ts | 27 +++--
.../metron-alerts/src/app/model/filter.spec.ts | 42 ++++++-
.../metron-alerts/src/app/model/filter.ts | 69 ++++++++---
.../metron-alerts/src/app/utils/constants.ts | 18 +--
.../metron-alerts/src/app/utils/utils.ts | 16 +--
7 files changed, 259 insertions(+), 50 deletions(-)
diff --git a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
index b12cb60..74fa468 100644
--- a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
+++ b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
@@ -227,8 +227,8 @@ export class AlertsListComponent implements OnInit, OnDestroy {
this.search();
}
- onSearch($event) {
- this.queryBuilder.setSearch($event);
+ onSearch(query: string) {
+ this.queryBuilder.setSearch(query);
this.timeStampfilterPresent = this.queryBuilder.isTimeStampFieldPresent();
this.search();
return false;
diff --git a/metron-interface/metron-alerts/src/app/alerts/alerts-list/query-builder.spec.ts b/metron-interface/metron-alerts/src/app/alerts/alerts-list/query-builder.spec.ts
new file mode 100644
index 0000000..b8e4ca9
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/alerts/alerts-list/query-builder.spec.ts
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { QueryBuilder } from './query-builder';
+import { Filter } from 'app/model/filter';
+import { TIMESTAMP_FIELD_NAME } from '../../utils/constants';
+import { Utils } from 'app/utils/utils';
+
+
+describe('query-builder', () => {
+
+ it('should be able to handle multiple filters', () => {
+ const queryBuilder = new QueryBuilder();
+
+ queryBuilder.setSearch('alert_status:RESOLVE AND ip_src_addr:0.0.0.0');
+
+ expect(queryBuilder.searchRequest.query).toBe(
+ '(alert_status:RESOLVE OR metron_alert.alert_status:RESOLVE) AND (ip_src_addr:0.0.0.0 OR metron_alert.ip_src_addr:0.0.0.0)'
+ );
+ });
+
+ it('should be able to handle multiple EXCLUDING filters for the same field', () => {
+ const queryBuilder = new QueryBuilder();
+
+ queryBuilder.setSearch('-alert_status:RESOLVE AND -alert_status:DISMISS');
+
+ expect(queryBuilder.searchRequest.query).toBe(
+ '-(alert_status:RESOLVE OR metron_alert.alert_status:RESOLVE) AND -(alert_status:DISMISS OR metron_alert.alert_status:DISMISS)'
+ );
+ });
+
+ it('should be able to handle group multiple clauses to a single field, aka. field grouping', () => {
+ const queryBuilder = new QueryBuilder();
+
+ queryBuilder.setSearch('alert_status:(RESOLVE OR DISMISS)');
+
+ expect(queryBuilder.searchRequest.query).toBe(
+ '(alert_status:(RESOLVE OR DISMISS) OR metron_alert.alert_status:(RESOLVE OR DISMISS))'
+ );
+ });
+
+ it('should trim whitespace', () => {
+ const queryBuilder = new QueryBuilder();
+
+ queryBuilder.setSearch(' alert_status:(RESOLVE OR DISMISS) ');
+
+ expect(queryBuilder.searchRequest.query).toBe(
+ '(alert_status:(RESOLVE OR DISMISS) OR metron_alert.alert_status:(RESOLVE OR DISMISS))'
+ );
+ });
+
+ it('should remove wildcard', () => {
+ const queryBuilder = new QueryBuilder();
+
+ queryBuilder.setSearch('* alert_status:(RESOLVE OR DISMISS)');
+
+ expect(queryBuilder.searchRequest.query).toBe(
+ '(alert_status:(RESOLVE OR DISMISS) OR metron_alert.alert_status:(RESOLVE OR DISMISS))'
+ );
+ });
+
+ it('should properly parse excluding filters event with wildcard and whitespaces', () => {
+ const queryBuilder = new QueryBuilder();
+
+ queryBuilder.setSearch('* -alert_status:(RESOLVE OR DISMISS)');
+
+ expect(queryBuilder.searchRequest.query).toBe(
+ '-(alert_status:(RESOLVE OR DISMISS) OR metron_alert.alert_status:(RESOLVE OR DISMISS))'
+ );
+ });
+
+ it('should remove wildcard from an excluding filter', () => {
+ const queryBuilder = new QueryBuilder();
+
+ queryBuilder.setSearch('* -alert_status:(RESOLVE OR DISMISS)');
+
+ expect(queryBuilder.searchRequest.query).toBe(
+ '-(alert_status:(RESOLVE OR DISMISS) OR metron_alert.alert_status:(RESOLVE OR DISMISS))'
+ );
+ });
+
+ it('should allow only one timerange filter', () => {
+ const queryBuilder = new QueryBuilder();
+
+ queryBuilder.addOrUpdateFilter(new Filter(TIMESTAMP_FIELD_NAME, '[1552863600000 TO 1552950000000]'));
+ queryBuilder.addOrUpdateFilter(new Filter(TIMESTAMP_FIELD_NAME, '[1552863700000 TO 1552960000000]'));
+
+ expect(queryBuilder.generateSelect()).toBe('(timestamp:[1552863700000 TO 1552960000000] OR ' +
+ 'metron_alert.timestamp:[1552863700000 TO 1552960000000])');
+ });
+
+ it('should escape : chars in ElasticSearch field names', () => {
+ const queryBuilder = new QueryBuilder();
+
+ queryBuilder.setSearch('source:type:bro');
+
+ expect(queryBuilder.searchRequest.query).toBe('(source\\:type:bro OR metron_alert.source\\:type:bro)');
+ });
+
+ it('should escape ALL : chars in ElasticSearch field names', () => {
+ const queryBuilder = new QueryBuilder();
+
+ queryBuilder.setSearch('enrichments:geo:ip_dst_addr:country:US');
+
+ expect(queryBuilder.searchRequest.query).toBe('(enrichments\\:geo\\:ip_dst_addr\\:country:US ' +
+ 'OR metron_alert.enrichments\\:geo\\:ip_dst_addr\\:country:US)');
+ });
+
+ it('should not multiply escaping in field name', () => {
+ const queryBuilder = new QueryBuilder();
+
+ queryBuilder.setSearch('source:type:bro');
+ queryBuilder.setSearch('source:type:bro');
+ queryBuilder.setSearch('source:type:bro');
+
+ expect(queryBuilder.searchRequest.query).toBe('(source\\:type:bro OR metron_alert.source\\:type:bro)');
+ });
+
+});
diff --git a/metron-interface/metron-alerts/src/app/alerts/alerts-list/query-builder.ts b/metron-interface/metron-alerts/src/app/alerts/alerts-list/query-builder.ts
index 06e6075..6cbed25 100644
--- a/metron-interface/metron-alerts/src/app/alerts/alerts-list/query-builder.ts
+++ b/metron-interface/metron-alerts/src/app/alerts/alerts-list/query-builder.ts
@@ -77,8 +77,17 @@ export class QueryBuilder {
addOrUpdateFilter(filter: Filter) {
let existingFilterIndex = -1;
+
+ // only one timerange filter applicable
+ if (filter.field === TIMESTAMP_FIELD_NAME) {
+ this.removeFilter(filter.field);
+ this._filters.push(filter);
+ this.onSearchChange();
+ return;
+ }
+
let existingFilter = this._filters.find((tFilter, index) => {
- if (tFilter.field === filter.field) {
+ if (filter.equals(tFilter)) {
existingFilterIndex = index;
return true;
}
@@ -152,24 +161,28 @@ export class QueryBuilder {
this.searchRequest.sort = [sortField];
}
- private updateFilters(tQuery: string, updateNameTransform = false) {
- let query = tQuery;
+ private updateFilters(query: string, updateNameTransform = false) {
this.removeDisplayedFilters();
if (query && query !== '' && query !== '*') {
let terms = query.split(' AND ');
for (let term of terms) {
- let separatorPos = term.lastIndexOf(':');
- let field = term.substring(0, separatorPos).replace('\\', '');
+ let [field, value] = this.splitTerm(term);
field = updateNameTransform ? ColumnNamesService.getColumnDisplayKey(field) : field;
- let value = term.substring(separatorPos + 1, term.length);
+ value = value.trim();
+
this.addOrUpdateFilter(new Filter(field, value));
}
}
}
+ private splitTerm(term): string[] {
+ const lastIdxOfSeparator = term.lastIndexOf(':');
+ return [ term.substring(0, lastIdxOfSeparator), term.substring(lastIdxOfSeparator + 1) ];
+ }
+
private removeDisplayedFilters() {
- for (let i = this._filters.length-1; i >= 0; i--) {
+ for (let i = this._filters.length - 1; i >= 0; i--) {
if (this._filters[i].display) {
this._filters.splice(i, 1);
}
diff --git a/metron-interface/metron-alerts/src/app/model/filter.spec.ts b/metron-interface/metron-alerts/src/app/model/filter.spec.ts
index d073ebd..b911974 100644
--- a/metron-interface/metron-alerts/src/app/model/filter.spec.ts
+++ b/metron-interface/metron-alerts/src/app/model/filter.spec.ts
@@ -51,8 +51,8 @@ describe('model.Filter', () => {
it('getQueryString for time range filter for display', () => {
const filter = new Filter(TIMESTAMP_FIELD_NAME, '[1552863600000 TO 1552950000000]', true);
- expect(filter.getQueryString()).toBe('(timestamp:\\[1552863600000\\ TO\\ 1552950000000\\] OR ' +
- 'metron_alert.timestamp:\\[1552863600000\\ TO\\ 1552950000000\\])');
+ expect(filter.getQueryString()).toBe('(timestamp:[1552863600000 TO 1552950000000] OR ' +
+ 'metron_alert.timestamp:[1552863600000 TO 1552950000000])');
});
/**
@@ -67,4 +67,42 @@ describe('model.Filter', () => {
filter.getQueryString();
expect(Utils.timeRangeToDateObj).toHaveBeenCalledWith(timeRange);
});
+
+ describe('equal function', () => {
+ it('should return false if field not equals', () => {
+ const filterA = new Filter('testField', 'someValue', false);
+ const filterB = new Filter('otherField', 'someValue', false);
+
+ expect(filterA.equals(filterB)).toBe(false);
+ });
+
+ it('should return false if value not equals', () => {
+ const filterA = new Filter('testField', 'someValue', false);
+ const filterB = new Filter('testField', 'otherValue', false);
+
+ expect(filterA.equals(filterB)).toBe(false);
+ });
+
+ it('should return true if both field and value are equals', () => {
+ const filterA = new Filter('testField', 'someValue', false);
+ const filterB = new Filter('testField', 'someValue', false);
+
+ expect(filterA.equals(filterB)).toBe(true);
+ });
+ })
+
+ it('excluding filtering', () => {
+ const filter = new Filter('-testField', 'someValue', false);
+ expect(filter.getQueryString()).toBe('-(testField:someValue OR metron_alert.testField:someValue)');
+ });
+
+ it('toJSON should return with a JSON representation of a Filter', () => {
+ const filter = new Filter('testField', 'someValue', false);
+ expect(filter.toJSON()).toEqual({ field: 'testField', value: 'someValue', display: false });
+ });
+
+ it('toJSON should return with a JSON representation of a Filter, including exclude operator in field value', () => {
+ const filter = new Filter('-testField', 'someValue', false);
+ expect(filter.toJSON()).toEqual({ field: '-testField', value: 'someValue', display: false });
+ });
});
diff --git a/metron-interface/metron-alerts/src/app/model/filter.ts b/metron-interface/metron-alerts/src/app/model/filter.ts
index 5d56c49..fb3f082 100644
--- a/metron-interface/metron-alerts/src/app/model/filter.ts
+++ b/metron-interface/metron-alerts/src/app/model/filter.ts
@@ -15,16 +15,25 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import {Utils} from '../utils/utils';
-import {TIMESTAMP_FIELD_NAME} from '../utils/constants';
-import {DateFilterValue} from './date-filter-value';
+import { Utils } from '../utils/utils';
+import { TIMESTAMP_FIELD_NAME, GIUD_FIELD_NAME } from '../utils/constants';
+import { DateFilterValue } from './date-filter-value';
export class Filter {
- field: string;
value: string;
display: boolean;
dateFilterValue: DateFilterValue;
+ private readonly excludeOperatorRxp = /^-/;
+ private readonly excludeOperator = '-';
+ private isExcluding = false;
+
+ private clearedField: string;
+
+ get field(): string {
+ return this.operatorToAdd() + this.clearedField;
+ }
+
static fromJSON(objs: Filter[]): Filter[] {
let filters = [];
if (objs) {
@@ -35,40 +44,68 @@ export class Filter {
return filters;
}
+ toJSON() {
+ return { field: this.field, value: this.value, display: this.display };
+ }
+
constructor(field: string, value: string, display = true) {
- this.field = field;
+ const { clearedField, isExcluding } = this.parseAndClearField(field);
+
this.value = value;
this.display = display;
+
+ this.clearedField = clearedField;
+ this.isExcluding = isExcluding;
+ }
+
+ private parseAndClearField(field: string): { clearedField: string, isExcluding: boolean } {
+ field = field.replace(/\*/, ''); // removing wildcard caracter
+ field = field.trim(); // removing whitespaces
+ const isExcluding = this.excludeOperatorRxp.test(field); // looking for excluding operator
+ const clearedField = field.replace(this.excludeOperatorRxp, ''); // removing exlude operator
+
+ return { clearedField, isExcluding };
}
getQueryString(): string {
- if (this.field === 'guid') {
+ if (this.clearedField === GIUD_FIELD_NAME) {
let valueWithQuote = '\"' + this.value + '\"';
- return this.createNestedQueryWithoutValueEscaping(this.field, valueWithQuote);
+ return this.createNestedQuery(this.clearedField, valueWithQuote);
}
- if (this.field === TIMESTAMP_FIELD_NAME && !this.display) {
+ if (this.clearedField === TIMESTAMP_FIELD_NAME && !this.display) {
this.dateFilterValue = Utils.timeRangeToDateObj(this.value);
if (this.dateFilterValue !== null && this.dateFilterValue.toDate !== null) {
- return this.createNestedQueryWithoutValueEscaping(this.field,
+ return this.createNestedQuery(this.clearedField,
'[' + this.dateFilterValue.fromDate + ' TO ' + this.dateFilterValue.toDate + ']');
} else {
- return this.createNestedQueryWithoutValueEscaping(this.field, this.value);
+ return this.createNestedQuery(this.clearedField, this.value);
}
}
- return this.createNestedQuery(this.field, this.value);
+ return this.createNestedQuery(this.clearedField, this.value);
}
private createNestedQuery(field: string, value: string): string {
+ field = this.escapingESSpearators(field);
+
+ return this.operatorToAdd() + '(' + field + ':' + value + ' OR ' +
+ this.addMetaAlertPrefix('metron_alert', field) + ':' + value + ')';
+ }
+
+ private escapingESSpearators(field: string): string {
+ return field.replace(/\:/g, '\\:');
+ }
- return '(' + Utils.escapeESField(field) + ':' + Utils.escapeESValue(value) + ' OR ' +
- Utils.escapeESField('metron_alert.' + field) + ':' + Utils.escapeESValue(value) + ')';
+ private operatorToAdd() {
+ return this.isExcluding ? this.excludeOperator : '';
}
- private createNestedQueryWithoutValueEscaping(field: string, value: string): string {
+ private addMetaAlertPrefix(prefix: string, field: string): string {
+ return prefix + '.' + field;
+ }
- return '(' + Utils.escapeESField(field) + ':' + value + ' OR ' +
- Utils.escapeESField('metron_alert.' + field) + ':' + value + ')';
+ equals(filter: Filter): boolean {
+ return this.field === filter.field && this.value === filter.value;
}
}
diff --git a/metron-interface/metron-alerts/src/app/utils/constants.ts b/metron-interface/metron-alerts/src/app/utils/constants.ts
index 7edddc1..929d140 100644
--- a/metron-interface/metron-alerts/src/app/utils/constants.ts
+++ b/metron-interface/metron-alerts/src/app/utils/constants.ts
@@ -26,17 +26,19 @@ export const ALERTS_SAVED_SEARCH = 'metron-alerts-saved-search';
export const ALERTS_TABLE_METADATA = 'metron-alerts-table-metadata';
export const ALERTS_COLUMN_NAMES = 'metron-alerts-column-names';
-export let TIMESTAMP_FIELD_NAME = 'timestamp';
-export let ALL_TIME = 'all-time';
+export const TIMESTAMP_FIELD_NAME = 'timestamp';
+export const GIUD_FIELD_NAME = 'guid';
-export let DEFAULT_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH:mm:ss';
-export let CUSTOMM_DATE_RANGE_LABEL = 'Date Range';
+export const ALL_TIME = 'all-time';
-export let TREE_SUB_GROUP_SIZE = 5;
-export let INDEXES = environment.indices ? environment.indices.split(',') : [];
-export let POLLING_DEFAULT_STATE = !environment.defaultPollingState;
+export const DEFAULT_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH:mm:ss';
+export const CUSTOMM_DATE_RANGE_LABEL = 'Date Range';
-export let MAX_ALERTS_IN_META_ALERTS = 350;
+export const TREE_SUB_GROUP_SIZE = 5;
+export const INDEXES = environment.indices ? environment.indices.split(',') : [];
+export const POLLING_DEFAULT_STATE = !environment.defaultPollingState;
+
+export const MAX_ALERTS_IN_META_ALERTS = 350;
export const DEFAULT_END_TIME = new Date();
export const DEFAULT_START_TIME = new Date().setDate(DEFAULT_END_TIME.getDate() - 5);
diff --git a/metron-interface/metron-alerts/src/app/utils/utils.ts b/metron-interface/metron-alerts/src/app/utils/utils.ts
index fbec2ed..ca2ad60 100644
--- a/metron-interface/metron-alerts/src/app/utils/utils.ts
+++ b/metron-interface/metron-alerts/src/app/utils/utils.ts
@@ -17,26 +17,12 @@
*/
import * as moment from 'moment/moment';
-import { DEFAULT_START_TIME, DEFAULT_END_TIME, DEFAULT_TIMESTAMP_FORMAT, META_ALERTS_SENSOR_TYPE } from './constants';
+import { DEFAULT_TIMESTAMP_FORMAT, META_ALERTS_SENSOR_TYPE } from './constants';
import { Alert } from '../model/alert';
import { DateFilterValue } from '../model/date-filter-value';
-import { PcapRequest } from '../pcap/model/pcap.request';
-import { PcapFilterFormValue } from '../pcap/pcap-filters/pcap-filters.component';
-import { FormGroup } from '@angular/forms';
export class Utils {
- public static escapeESField(field: string): string {
- return field.replace(/:/g, '\\:');
- }
-
- public static escapeESValue(value: string): string {
- return String(value)
- .replace(/[\*\+\-=~><\"\?^\${}\(\)\:\!\/[\]\\\s]/g, '\\$&') // replace single special characters
- .replace(/\|\|/g, '\\||') // replace ||
- .replace(/\&\&/g, '\\&&'); // replace &&
- }
-
public static getAlertSensorType(alert: Alert, sourceType: string): string {
if (alert.source[sourceType] && alert.source[sourceType].length > 0) {
return alert.source[sourceType];