You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2019/04/02 17:00:26 UTC
[metron] branch master updated: METRON-2039 Time range queries do
not work with Solr (merrimanr) closes apache/metron#1359
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 4da8990 METRON-2039 Time range queries do not work with Solr (merrimanr) closes apache/metron#1359
4da8990 is described below
commit 4da899043ffdb222f8a928cd30bc0f47b99c632a
Author: merrimanr <me...@gmail.com>
AuthorDate: Tue Apr 2 11:50:36 2019 -0500
METRON-2039 Time range queries do not work with Solr (merrimanr) closes apache/metron#1359
---
.../metron-alerts/src/app/model/filter.spec.ts | 70 +++++++++++++++++++
.../metron-alerts/src/app/model/filter.ts | 2 +-
.../shared/time-range/time-range.component.spec.ts | 80 +++++++++++++++++++++-
.../app/shared/time-range/time-range.component.ts | 6 +-
.../metron-alerts/src/app/utils/utils.spec.ts | 76 ++++++++++++++++++++
.../metron-alerts/src/app/utils/utils.ts | 6 +-
6 files changed, 232 insertions(+), 8 deletions(-)
diff --git a/metron-interface/metron-alerts/src/app/model/filter.spec.ts b/metron-interface/metron-alerts/src/app/model/filter.spec.ts
new file mode 100644
index 0000000..d073ebd
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/model/filter.spec.ts
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Filter } from './filter';
+import { TIMESTAMP_FIELD_NAME } from 'app/utils/constants';
+import { Utils } from 'app/utils/utils';
+
+describe('model.Filter', () => {
+
+ const expectedTimeRangeQueryString = '(timestamp:[1552863600000 TO 1552950000000] OR ' +
+ 'metron_alert.timestamp:[1552863600000 TO 1552950000000])';
+
+ it('should have getQueryString function', () => {
+ const filter = new Filter('testField', 'someValue', false);
+ expect(typeof filter.getQueryString).toBe('function');
+ });
+
+ it('getQueryString for basic filter', () => {
+ const filter = new Filter('testField', 'someValue', false);
+ expect(filter.getQueryString()).toBe('(testField:someValue OR metron_alert.testField:someValue)');
+ });
+
+ it('getQueryString for guid filter', () => {
+ const filter = new Filter('guid', 'someValue', false);
+ expect(filter.getQueryString()).toBe('(guid:"someValue" OR metron_alert.guid:"someValue")');
+ });
+
+ it('getQueryString for time range filter with []', () => {
+ const filter = new Filter(TIMESTAMP_FIELD_NAME, '[1552863600000 TO 1552950000000]', false);
+ expect(filter.getQueryString()).toBe(expectedTimeRangeQueryString);
+ });
+
+ it('getQueryString for time range filter without []', () => {
+ const filter = new Filter(TIMESTAMP_FIELD_NAME, '1552863600000 TO 1552950000000', false);
+ expect(filter.getQueryString()).toBe(expectedTimeRangeQueryString);
+ });
+
+ it('getQueryString for time range filter for display', () => {
+ const filter = new Filter(TIMESTAMP_FIELD_NAME, '[1552863600000 TO 1552950000000]', true);
+ expect(filter.getQueryString()).toBe('(timestamp:\\[1552863600000\\ TO\\ 1552950000000\\] OR ' +
+ 'metron_alert.timestamp:\\[1552863600000\\ TO\\ 1552950000000\\])');
+ });
+
+ /**
+ * Actual time range and quick range conversion tested in utils/utils.spec.ts
+ * until further refactoring.
+ */
+ it('getQueryString for time range filter should call Utils.timeRangeToDateObj', () => {
+ const timeRange = '1552863600000 TO 1552950000000';
+ spyOn(Utils, 'timeRangeToDateObj').and.callThrough();
+ const filter = new Filter(TIMESTAMP_FIELD_NAME, timeRange, false);
+
+ filter.getQueryString();
+ expect(Utils.timeRangeToDateObj).toHaveBeenCalledWith(timeRange);
+ });
+});
diff --git a/metron-interface/metron-alerts/src/app/model/filter.ts b/metron-interface/metron-alerts/src/app/model/filter.ts
index c90c7a6..5d56c49 100644
--- a/metron-interface/metron-alerts/src/app/model/filter.ts
+++ b/metron-interface/metron-alerts/src/app/model/filter.ts
@@ -51,7 +51,7 @@ export class Filter {
this.dateFilterValue = Utils.timeRangeToDateObj(this.value);
if (this.dateFilterValue !== null && this.dateFilterValue.toDate !== null) {
return this.createNestedQueryWithoutValueEscaping(this.field,
- '(>=' + this.dateFilterValue.fromDate + ' AND ' + ' <=' + this.dateFilterValue.toDate + ')');
+ '[' + this.dateFilterValue.fromDate + ' TO ' + this.dateFilterValue.toDate + ']');
} else {
return this.createNestedQueryWithoutValueEscaping(this.field, this.value);
}
diff --git a/metron-interface/metron-alerts/src/app/shared/time-range/time-range.component.spec.ts b/metron-interface/metron-alerts/src/app/shared/time-range/time-range.component.spec.ts
index 4b33b1d..51c64b2 100644
--- a/metron-interface/metron-alerts/src/app/shared/time-range/time-range.component.spec.ts
+++ b/metron-interface/metron-alerts/src/app/shared/time-range/time-range.component.spec.ts
@@ -22,6 +22,10 @@ import { TimeRangeComponent } from './time-range.component';
import { DatePickerComponent } from '../date-picker/date-picker.component';
import { MapKeysPipe } from '../pipes/map-keys.pipe';
import { FormsModule } from '@angular/forms';
+import { By } from '@angular/platform-browser';
+import { Filter } from 'app/model/filter';
+import { TIMESTAMP_FIELD_NAME } from 'app/utils/constants';
+import { DateFilterValue } from 'app/model/date-filter-value';
describe('TimeRangeComponent', () => {
let component: TimeRangeComponent;
@@ -32,7 +36,7 @@ describe('TimeRangeComponent', () => {
imports: [
FormsModule
],
- declarations: [
+ declarations: [
TimeRangeComponent,
DatePickerComponent,
MapKeysPipe
@@ -50,4 +54,78 @@ describe('TimeRangeComponent', () => {
it('should be created', () => {
expect(component).toBeTruthy();
});
+
+ describe('From/To time ranges', () => {
+
+ it('should use date pickers to set range', () => {
+ component.datePickerFromDate = '2000-01-31 00:00:00';
+ component.datePickerToDate = '2000-02-28 00:00:00';
+
+ spyOn(component.timeRangeChange, 'emit');
+ component.applyCustomDate();
+
+ const filter = new Filter(TIMESTAMP_FIELD_NAME, '[949273200000 TO 951692400000]', false);
+ filter.dateFilterValue = new DateFilterValue(949273200000, 951692400000);
+
+ expect(component.timeRangeChange.emit).toHaveBeenCalledWith(filter);
+ });
+
+ it('should apply current date and time if To field empty', () => {
+ const currentTs = new Date().getTime();
+
+ component.datePickerFromDate = '2000-01-31 00:00:00';
+ component.datePickerToDate = '';
+
+ spyOn(component.timeRangeChange, 'emit');
+ component.applyCustomDate();
+
+ const filter = new Filter(TIMESTAMP_FIELD_NAME, `[949273200000 TO ${currentTs}]`, false);
+ filter.dateFilterValue = new DateFilterValue(949273200000, currentTs);
+
+ expect(component.timeRangeChange.emit).toHaveBeenCalledWith(filter);
+ });
+
+
+ });
+
+ describe('Quick Ranges', () => {
+ [
+ { label: 'Last 7 days', rangeId: 'last-7-days' },
+ { label: 'Last 30 days', rangeId: 'last-30-days' },
+ { label: 'Last 60 days', rangeId: 'last-60-days' },
+ { label: 'Last 90 days', rangeId: 'last-90-days' },
+ { label: 'Last 6 months', rangeId: 'last-6-months' },
+ { label: 'Last 1 year', rangeId: 'last-1-year' },
+ { label: 'Last 2 years', rangeId: 'last-2-years' },
+ { label: 'Last 5 years', rangeId: 'last-5-years' },
+ { label: 'Yesterday', rangeId: 'yesterday' },
+ { label: 'Day before yesterday', rangeId: 'day-before-yesterday' },
+ { label: 'This day last week', rangeId: 'this-day-last-week' },
+ { label: 'Previous week', rangeId: 'previous-week' },
+ { label: 'Previous month', rangeId: 'previous-month' },
+ { label: 'Previous year', rangeId: 'previous-year' },
+ { label: 'All time', rangeId: 'all-time' },
+ { label: 'Today', rangeId: 'today' },
+ { label: 'Today so far', rangeId: 'today-so-far' },
+ { label: 'This week', rangeId: 'this-week' },
+ { label: 'This week so far', rangeId: 'this-week-so-far' },
+ { label: 'This month', rangeId: 'this-month' },
+ { label: 'This year', rangeId: 'this-year' },
+ { label: 'Last 5 minutes', rangeId: 'last-5-minutes' },
+ { label: 'Last 15 minutes', rangeId: 'last-15-minutes' },
+ { label: 'Last 30 minutes', rangeId: 'last-30-minutes' },
+ { label: 'Last 1 hour', rangeId: 'last-1-hour' },
+ { label: 'Last 3 hours', rangeId: 'last-3-hours' },
+ { label: 'Last 6 hours', rangeId: 'last-6-hours' },
+ { label: 'Last 12 hours', rangeId: 'last-12-hours' },
+ { label: 'Last 24 hours', rangeId: 'last-24-hours' },
+ ].forEach((test) => {
+ it(`Clicking on ${test.label} should emit timeRangeChange event with range identifier: "${test.rangeId}"`, () => {
+ spyOn(component.timeRangeChange, 'emit');
+ fixture.debugElement.query(By.css(`span[id="${test.rangeId}"]`)).nativeElement.click();
+
+ expect(component.timeRangeChange.emit).toHaveBeenCalledWith(new Filter(TIMESTAMP_FIELD_NAME, test.rangeId, false));
+ })
+ });
+ });
});
diff --git a/metron-interface/metron-alerts/src/app/shared/time-range/time-range.component.ts b/metron-interface/metron-alerts/src/app/shared/time-range/time-range.component.ts
index 9c3a61f..2a989d4 100644
--- a/metron-interface/metron-alerts/src/app/shared/time-range/time-range.component.ts
+++ b/metron-interface/metron-alerts/src/app/shared/time-range/time-range.component.ts
@@ -166,11 +166,11 @@ export class TimeRangeComponent implements OnChanges {
this.toDateStr = this.datePickerToDate.length > 0 ? moment(this.datePickerToDate).format(DEFAULT_TIMESTAMP_FORMAT) : 'now';
this.fromDateStr = moment(this.datePickerFromDate).format(DEFAULT_TIMESTAMP_FORMAT);
- let toDate = this.datePickerToDate.length > 0 ? new Date(this.toDateStr).getTime() : null;
+ let toDate = this.datePickerToDate.length > 0 ? new Date(this.toDateStr).getTime() : new Date().getTime();
let fromDate = new Date(this.fromDateStr).getTime();
- let toDateExpression = this.datePickerToDate.length > 0 ? (' AND ' + ' <=' + toDate) : '';
+ let toDateExpression = ' TO ' + toDate;
- let value = '(>=' + fromDate + toDateExpression + ')';
+ let value = '[' + fromDate + toDateExpression + ']';
let filter = new Filter(TIMESTAMP_FIELD_NAME, value, false);
filter.dateFilterValue = new DateFilterValue(fromDate, toDate);
this.timeRangeChange.emit(filter);
diff --git a/metron-interface/metron-alerts/src/app/utils/utils.spec.ts b/metron-interface/metron-alerts/src/app/utils/utils.spec.ts
new file mode 100644
index 0000000..b64c461
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/utils/utils.spec.ts
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Utils } from './utils';
+import { DateFilterValue } from 'app/model/date-filter-value';
+
+fdescribe('utils.Utils', () => {
+
+ it('Converting time range based on From/To', () => {
+ expect(Utils.timeRangeToDateObj('949273200000 TO 951692400000')).toEqual(new DateFilterValue(949273200000, 951692400000));
+ });
+
+ describe('Converting quick range to date object', () => {
+ const getTestObj = (label, rangeId, from, to) => {
+ return {
+ label,
+ rangeId,
+ from: new Date(from).getTime(),
+ to: new Date(to).getTime(),
+ }
+ }
+
+ [
+ getTestObj('Last 7 days', 'last-7-days', '1999-12-25T12:00:00', '2000-01-01T12:00:00'),
+ getTestObj('Last 30 days', 'last-30-days', '1999-12-02T12:00:00', '2000-01-01T12:00:00'),
+ getTestObj('Last 60 days', 'last-60-days', '1999-11-02T12:00:00', '2000-01-01T12:00:00'),
+ getTestObj('Last 90 days', 'last-90-days', '1999-10-03T12:00:00', '2000-01-01T12:00:00'),
+ getTestObj('Last 6 months', 'last-6-months', '1999-07-01T12:00:00', '2000-01-01T12:00:00'),
+ getTestObj('Last 1 year', 'last-1-year', '1999-01-01T12:00:00', '2000-01-01T12:00:00'),
+ getTestObj('Last 2 years', 'last-2-years', '1998-01-01T12:00:00', '2000-01-01T12:00:00'),
+ getTestObj('Last 5 years', 'last-5-years', '1995-01-01T12:00:00', '2000-01-01T12:00:00'),
+ getTestObj('Yesterday', 'yesterday', '1999-12-31T00:00:00', '1999-12-31T23:59:59'),
+ getTestObj('Day before yesterday', 'day-before-yesterday', '1999-12-30T00:00:00', '1999-12-30T23:59:59'),
+ getTestObj('This day last week', 'this-day-last-week', '1999-12-25T00:00:00', '1999-12-25T23:59:59'),
+ getTestObj('Previous week', 'previous-week', '1999-12-19T00:00:00', '1999-12-25T23:59:59'),
+ getTestObj('Previous month', 'previous-month', '1999-12-01T00:00:00', '1999-12-31T23:59:59'),
+ getTestObj('Previous year', 'previous-year', '1999-01-01T00:00:00', '1999-12-31T23:59:59'),
+ getTestObj('All time', 'all-time', '1970-01-01T05:30:00+05:30', '2100-01-01T05:30:00+05:30'),
+ getTestObj('Today', 'today', '2000-01-01T00:00:00', '2000-01-01T23:59:59'),
+ getTestObj('Today so far', 'today-so-far', '2000-01-01T00:00:00', '2000-01-01T12:00:00'),
+ getTestObj('This week', 'this-week', '1999-12-26T00:00:00', '2000-01-01T23:59:59'),
+ getTestObj('This week so far', 'this-week-so-far', '1999-12-26T00:00:00', '2000-01-01T12:00:00'),
+ getTestObj('This month', 'this-month', '2000-01-01T00:00:00', '2000-01-31T23:59:59'),
+ getTestObj('This year', 'this-year', '2000-01-01T00:00:00', '2000-12-31T23:59:59'),
+ getTestObj('Last 5 minutes', 'last-5-minutes', '2000-01-01T11:55:00', '2000-01-01T12:00:00'),
+ getTestObj('Last 15 minutes', 'last-15-minutes', '2000-01-01T11:45:00', '2000-01-01T12:00:00'),
+ getTestObj('Last 30 minutes', 'last-30-minutes', '2000-01-01T11:30:00', '2000-01-01T12:00:00'),
+ getTestObj('Last 1 hour', 'last-1-hour', '2000-01-01T11:00:00', '2000-01-01T12:00:00'),
+ getTestObj('Last 3 hours', 'last-3-hours', '2000-01-01T09:00:00', '2000-01-01T12:00:00'),
+ getTestObj('Last 6 hours', 'last-6-hours', '2000-01-01T06:00:00', '2000-01-01T12:00:00'),
+ getTestObj('Last 12 hours', 'last-12-hours', '2000-01-01T00:00:00', '2000-01-01T12:00:00'),
+ getTestObj('Last 24 hours', 'last-24-hours', '1999-12-31T12:00:00', '2000-01-01T12:00:00'),
+ ].forEach((test) => {
+ it(`Converting ${test.label} to DateFilterValue`, () => {
+ jasmine.clock().mockDate(new Date('2000-01-01T12:00:00'));
+
+ expect(Utils.timeRangeToDateObj(test.rangeId)).toEqual(new DateFilterValue(test.from, test.to));
+ })
+ });
+
+ });
+});
diff --git a/metron-interface/metron-alerts/src/app/utils/utils.ts b/metron-interface/metron-alerts/src/app/utils/utils.ts
index 872d33d..fbec2ed 100644
--- a/metron-interface/metron-alerts/src/app/utils/utils.ts
+++ b/metron-interface/metron-alerts/src/app/utils/utils.ts
@@ -61,9 +61,9 @@ export class Utils {
}
public static parseTimeRange(range: string): { toDate: number, fromDate: number } {
- let parsed = range.replace(/^\(>=/, '')
- .replace(/\)$/, '')
- .replace(/<=/, '').split('AND');
+ let parsed = range.replace(/^\[/, '')
+ .replace(/]$/, '')
+ .split('TO');
if (parsed.length === 2 && !isNaN(Number(parsed[0])) && !isNaN(Number(parsed[1]))) {
return {toDate: Number(parsed[1]), fromDate: Number(parsed[0])};
}