You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by su...@apache.org on 2021/03/11 04:00:23 UTC
[druid] branch master updated: Make web console fast around
sys.segments (#10909)
This is an automated email from the ASF dual-hosted git repository.
suneet pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 4897731 Make web console fast around sys.segments (#10909)
4897731 is described below
commit 4897731e376b80c37ea7713419bd89c7835809b9
Author: Vadim Ogievetsky <va...@ogievetsky.com>
AuthorDate: Wed Mar 10 19:59:50 2021 -0800
Make web console fast around sys.segments (#10909)
* do not load all the segments
* fix filtering
* update datasource view
* updated tests
* remove trimmedSegments
* Availability detail
* be smart about when showing smart modes
* fix tests
* add coordinator overlord mode
---
.../__snapshots__/header-bar.spec.tsx.snap | 36 ++
.../src/components/header-bar/header-bar.tsx | 59 ++-
.../table-column-selector.tsx | 18 +-
web-console/src/singletons/api.ts | 2 +-
web-console/src/utils/capabilities.ts | 10 +
web-console/src/utils/general.tsx | 9 +-
web-console/src/utils/local-storage-keys.tsx | 5 +
.../__snapshots__/datasource-view.spec.tsx.snap | 13 +-
.../src/views/datasource-view/datasource-view.tsx | 256 ++++++-----
.../__snapshots__/segments-view.spec.tsx.snap | 15 +
.../src/views/segments-view/segments-view.tsx | 485 +++++++++++----------
11 files changed, 566 insertions(+), 342 deletions(-)
diff --git a/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap b/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap
index 0aadb18..c2448c7 100644
--- a/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap
+++ b/web-console/src/components/header-bar/__snapshots__/header-bar.spec.tsx.snap
@@ -119,6 +119,42 @@ exports[`header bar matches snapshot 1`] = `
shouldDismissPopover={true}
text="Lookups"
/>
+ <Blueprint3.MenuDivider />
+ <Blueprint3.MenuItem
+ disabled={false}
+ icon="cog"
+ multiline={false}
+ popoverProps={Object {}}
+ shouldDismissPopover={true}
+ text="Console options"
+ >
+ <React.Fragment>
+ <Blueprint3.MenuItem
+ disabled={false}
+ multiline={false}
+ onClick={[Function]}
+ popoverProps={Object {}}
+ shouldDismissPopover={true}
+ text="Force Coordinator/Overlord mode"
+ />
+ <Blueprint3.MenuItem
+ disabled={false}
+ multiline={false}
+ onClick={[Function]}
+ popoverProps={Object {}}
+ shouldDismissPopover={true}
+ text="Force Coordinator mode"
+ />
+ <Blueprint3.MenuItem
+ disabled={false}
+ multiline={false}
+ onClick={[Function]}
+ popoverProps={Object {}}
+ shouldDismissPopover={true}
+ text="Force Overlord mode"
+ />
+ </React.Fragment>
+ </Blueprint3.MenuItem>
</Blueprint3.Menu>
}
defaultIsOpen={false}
diff --git a/web-console/src/components/header-bar/header-bar.tsx b/web-console/src/components/header-bar/header-bar.tsx
index 76bb378..b068e66 100644
--- a/web-console/src/components/header-bar/header-bar.tsx
+++ b/web-console/src/components/header-bar/header-bar.tsx
@@ -22,6 +22,7 @@ import {
Button,
Intent,
Menu,
+ MenuDivider,
MenuItem,
Navbar,
NavbarDivider,
@@ -39,12 +40,20 @@ import {
OverlordDynamicConfigDialog,
} from '../../dialogs';
import { getLink } from '../../links';
-import { Capabilities } from '../../utils';
+import {
+ Capabilities,
+ localStorageGetJson,
+ LocalStorageKeys,
+ localStorageRemove,
+ localStorageSetJson,
+} from '../../utils';
import { ExternalLink } from '../external-link/external-link';
import { PopoverText } from '../popover-text/popover-text';
import './header-bar.scss';
+const capabilitiesOverride = localStorageGetJson(LocalStorageKeys.CAPABILITIES_OVERRIDE);
+
export type HeaderActiveTab =
| null
| 'load-data'
@@ -121,6 +130,17 @@ const RestrictedMode = React.memo(function RestrictedMode(props: RestrictedModeP
);
break;
+ case 'coordinator-overlord':
+ label = 'Coordinator/Overlord mode';
+ message = (
+ <p>
+ It appears that you are accessing the console on the Coordinator/Overlord shared service.
+ Due to the lack of access to some APIs on this service the console will operate in a
+ limited mode. The full version of the console can be accessed on the Router service.
+ </p>
+ );
+ break;
+
case 'coordinator':
label = 'Coordinator mode';
message = (
@@ -216,6 +236,16 @@ export const HeaderBar = React.memo(function HeaderBar(props: HeaderBarProps) {
</Menu>
);
+ function setForcedMode(capabilities: Capabilities | undefined): void {
+ if (capabilities) {
+ localStorageSetJson(LocalStorageKeys.CAPABILITIES_OVERRIDE, capabilities);
+ } else {
+ localStorageRemove(LocalStorageKeys.CAPABILITIES_OVERRIDE);
+ }
+ location.reload();
+ }
+
+ const capabilitiesMode = capabilities.getModeExtended();
const configMenu = (
<Menu>
<MenuItem
@@ -243,6 +273,33 @@ export const HeaderBar = React.memo(function HeaderBar(props: HeaderBarProps) {
href="#lookups"
disabled={!capabilities.hasCoordinatorAccess()}
/>
+ <MenuDivider />
+ <MenuItem icon={IconNames.COG} text="Console options">
+ {capabilitiesOverride ? (
+ <MenuItem text="Clear forced mode" onClick={() => setForcedMode(undefined)} />
+ ) : (
+ <>
+ {capabilitiesMode !== 'coordinator-overlord' && (
+ <MenuItem
+ text="Force Coordinator/Overlord mode"
+ onClick={() => setForcedMode(Capabilities.COORDINATOR_OVERLORD)}
+ />
+ )}
+ {capabilitiesMode !== 'coordinator' && (
+ <MenuItem
+ text="Force Coordinator mode"
+ onClick={() => setForcedMode(Capabilities.COORDINATOR)}
+ />
+ )}
+ {capabilitiesMode !== 'overlord' && (
+ <MenuItem
+ text="Force Overlord mode"
+ onClick={() => setForcedMode(Capabilities.OVERLORD)}
+ />
+ )}
+ </>
+ )}
+ </MenuItem>
</Menu>
);
diff --git a/web-console/src/components/table-column-selector/table-column-selector.tsx b/web-console/src/components/table-column-selector/table-column-selector.tsx
index 6cdd951..5ce6db5 100644
--- a/web-console/src/components/table-column-selector/table-column-selector.tsx
+++ b/web-console/src/components/table-column-selector/table-column-selector.tsx
@@ -18,7 +18,7 @@
import { Button, Menu, Popover, Position } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
-import React from 'react';
+import React, { useState } from 'react';
import { MenuCheckbox } from '../menu-checkbox/menu-checkbox';
@@ -27,13 +27,15 @@ import './table-column-selector.scss';
interface TableColumnSelectorProps {
columns: string[];
onChange: (column: string) => void;
+ onClose?: (added: number) => void;
tableColumnsHidden: string[];
}
export const TableColumnSelector = React.memo(function TableColumnSelector(
props: TableColumnSelectorProps,
) {
- const { columns, onChange, tableColumnsHidden } = props;
+ const { columns, onChange, onClose, tableColumnsHidden } = props;
+ const [added, setAdded] = useState(0);
const isColumnShown = (column: string) => !tableColumnsHidden.includes(column);
@@ -44,7 +46,12 @@ export const TableColumnSelector = React.memo(function TableColumnSelector(
text={column}
key={column}
checked={isColumnShown(column)}
- onChange={() => onChange(column)}
+ onChange={() => {
+ if (!isColumnShown(column)) {
+ setAdded(added + 1);
+ }
+ onChange(column);
+ }}
/>
))}
</Menu>
@@ -57,6 +64,11 @@ export const TableColumnSelector = React.memo(function TableColumnSelector(
className="table-column-selector"
content={checkboxes}
position={Position.BOTTOM_RIGHT}
+ onOpened={() => setAdded(0)}
+ onClose={() => {
+ if (!onClose) return;
+ onClose(added);
+ }}
>
<Button rightIcon={IconNames.CARET_DOWN}>
Columns <span className="counter">{counterText}</span>
diff --git a/web-console/src/singletons/api.ts b/web-console/src/singletons/api.ts
index 7a05bdd..e14a13b 100644
--- a/web-console/src/singletons/api.ts
+++ b/web-console/src/singletons/api.ts
@@ -47,7 +47,7 @@ export class Api {
static encodePath(path: string): string {
return path.replace(
- /[?#%&'\[\]]/g,
+ /[?#%&'\[\]\\]/g,
c =>
'%' +
c
diff --git a/web-console/src/utils/capabilities.ts b/web-console/src/utils/capabilities.ts
index 96c01ab..624ad35 100644
--- a/web-console/src/utils/capabilities.ts
+++ b/web-console/src/utils/capabilities.ts
@@ -27,6 +27,7 @@ export type CapabilitiesModeExtended =
| 'no-sql'
| 'no-proxy'
| 'no-sql-no-proxy'
+ | 'coordinator-overlord'
| 'coordinator'
| 'overlord';
@@ -41,6 +42,7 @@ export interface CapabilitiesOptions {
export class Capabilities {
static STATUS_TIMEOUT = 2000;
static FULL: Capabilities;
+ static COORDINATOR_OVERLORD: Capabilities;
static COORDINATOR: Capabilities;
static OVERLORD: Capabilities;
@@ -154,6 +156,9 @@ export class Capabilities {
return 'no-sql-no-proxy';
}
} else {
+ if (coordinator && overlord) {
+ return 'coordinator-overlord';
+ }
if (coordinator) {
return 'coordinator';
}
@@ -198,6 +203,11 @@ Capabilities.FULL = new Capabilities({
coordinator: true,
overlord: true,
});
+Capabilities.COORDINATOR_OVERLORD = new Capabilities({
+ queryType: 'none',
+ coordinator: true,
+ overlord: true,
+});
Capabilities.COORDINATOR = new Capabilities({
queryType: 'none',
coordinator: true,
diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx
index 8783f08..672c176 100644
--- a/web-console/src/utils/general.tsx
+++ b/web-console/src/utils/general.tsx
@@ -98,7 +98,8 @@ interface NeedleAndMode {
mode: 'exact' | 'includes';
}
-function getNeedleAndMode(input: string): NeedleAndMode {
+export function getNeedleAndMode(filter: Filter): NeedleAndMode {
+ const input = filter.value.toLowerCase();
if (input.startsWith(`"`) && input.endsWith(`"`)) {
return {
needle: input.slice(1, -1),
@@ -114,7 +115,7 @@ function getNeedleAndMode(input: string): NeedleAndMode {
export function booleanCustomTableFilter(filter: Filter, value: any): boolean {
if (value == null) return false;
const haystack = String(value).toLowerCase();
- const needleAndMode: NeedleAndMode = getNeedleAndMode(filter.value.toLowerCase());
+ const needleAndMode: NeedleAndMode = getNeedleAndMode(filter);
const needle = needleAndMode.needle;
if (needleAndMode.mode === 'exact') {
return needle === haystack;
@@ -123,13 +124,13 @@ export function booleanCustomTableFilter(filter: Filter, value: any): boolean {
}
export function sqlQueryCustomTableFilter(filter: Filter): SqlExpression {
- const needleAndMode: NeedleAndMode = getNeedleAndMode(filter.value);
+ const needleAndMode: NeedleAndMode = getNeedleAndMode(filter);
const needle = needleAndMode.needle;
if (needleAndMode.mode === 'exact') {
return SqlRef.columnWithQuotes(filter.id).equal(SqlLiteral.create(needle));
} else {
return SqlFunction.simple('LOWER', [SqlRef.columnWithQuotes(filter.id)]).like(
- SqlLiteral.create(`%${needle.toLowerCase()}%`),
+ SqlLiteral.create(`%${needle}%`),
);
}
}
diff --git a/web-console/src/utils/local-storage-keys.tsx b/web-console/src/utils/local-storage-keys.tsx
index 7ae1c68..99e788c 100644
--- a/web-console/src/utils/local-storage-keys.tsx
+++ b/web-console/src/utils/local-storage-keys.tsx
@@ -67,3 +67,8 @@ export function localStorageGetJson(key: LocalStorageKeys): any {
return;
}
}
+
+export function localStorageRemove(key: LocalStorageKeys): void {
+ if (typeof localStorage === 'undefined') return;
+ return localStorage.removeItem(key);
+}
diff --git a/web-console/src/views/datasource-view/__snapshots__/datasource-view.spec.tsx.snap b/web-console/src/views/datasource-view/__snapshots__/datasource-view.spec.tsx.snap
index 947e1d2..866ea3d 100755
--- a/web-console/src/views/datasource-view/__snapshots__/datasource-view.spec.tsx.snap
+++ b/web-console/src/views/datasource-view/__snapshots__/datasource-view.spec.tsx.snap
@@ -26,7 +26,7 @@ exports[`data source view matches snapshot 1`] = `
}
>
<Blueprint3.MenuItem
- disabled={false}
+ disabled={true}
icon="application"
multiline={false}
onClick={[Function]}
@@ -61,7 +61,7 @@ exports[`data source view matches snapshot 1`] = `
Array [
"Datasource name",
"Availability",
- "Segment load/drop queues",
+ "Availability detail",
"Total data size",
"Segment size",
"Segment granularity",
@@ -76,6 +76,7 @@ exports[`data source view matches snapshot 1`] = `
]
}
onChange={[Function]}
+ onClose={[Function]}
tableColumnsHidden={Array []}
/>
</Memo(ViewControlBar)>
@@ -150,9 +151,8 @@ exports[`data source view matches snapshot 1`] = `
Object {
"Cell": [Function],
"Header": "Availability",
- "accessor": [Function],
+ "accessor": "num_segments",
"filterable": false,
- "id": "availability",
"minWidth": 200,
"show": true,
"sortMethod": [Function],
@@ -160,13 +160,12 @@ exports[`data source view matches snapshot 1`] = `
Object {
"Cell": [Function],
"Header": <React.Fragment>
- Segment load/drop
+ Availability
<br />
- queues
+ detail
</React.Fragment>,
"accessor": "num_segments_to_load",
"filterable": false,
- "id": "load-drop",
"minWidth": 100,
"show": true,
},
diff --git a/web-console/src/views/datasource-view/datasource-view.tsx b/web-console/src/views/datasource-view/datasource-view.tsx
index adecc91..7e92dfa 100644
--- a/web-console/src/views/datasource-view/datasource-view.tsx
+++ b/web-console/src/views/datasource-view/datasource-view.tsx
@@ -49,6 +49,7 @@ import {
addFilter,
Capabilities,
CapabilitiesMode,
+ compact,
countBy,
deepGet,
formatBytes,
@@ -73,7 +74,7 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
full: [
'Datasource name',
'Availability',
- 'Segment load/drop queues',
+ 'Availability detail',
'Total data size',
'Segment size',
'Segment granularity',
@@ -89,7 +90,7 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
'no-sql': [
'Datasource name',
'Availability',
- 'Segment load/drop queues',
+ 'Availability detail',
'Total data size',
'Compaction',
'% Compacted',
@@ -100,7 +101,7 @@ const tableColumns: Record<CapabilitiesMode, string[]> = {
'no-proxy': [
'Datasource name',
'Availability',
- 'Segment load/drop queues',
+ 'Availability detail',
'Total data size',
'Segment size',
'Segment granularity',
@@ -150,7 +151,6 @@ const PERCENT_BRACES = [formatPercent(1)];
interface DatasourceQueryResultRow {
readonly datasource: string;
readonly num_segments: number;
- readonly num_available_segments: number;
readonly num_segments_to_load: number;
readonly num_segments_to_drop: number;
readonly minute_aligned_segments: number;
@@ -233,6 +233,12 @@ export interface DatasourcesViewState {
actions: BasicAction[];
}
+interface DatasourceQuery {
+ capabilities: Capabilities;
+ hiddenColumns: LocalStorageBackedArray<string>;
+ showUnused: boolean;
+}
+
export class DatasourcesView extends React.PureComponent<
DatasourcesViewProps,
DatasourcesViewState
@@ -241,34 +247,49 @@ export class DatasourcesView extends React.PureComponent<
static FULLY_AVAILABLE_COLOR = '#57d500';
static PARTIALLY_AVAILABLE_COLOR = '#ffbf00';
- static DATASOURCE_SQL = `SELECT
- datasource,
- COUNT(*) FILTER (WHERE (is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AS num_segments,
- COUNT(*) FILTER (WHERE is_available = 1 AND ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1)) AS num_available_segments,
- COUNT(*) FILTER (WHERE is_published = 1 AND is_overshadowed = 0 AND is_available = 0) AS num_segments_to_load,
- COUNT(*) FILTER (WHERE is_available = 1 AND NOT ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1)) AS num_segments_to_drop,
- COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%:00.000Z' AND "end" LIKE '%:00.000Z') AS minute_aligned_segments,
- COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%:00:00.000Z' AND "end" LIKE '%:00:00.000Z') AS hour_aligned_segments,
- COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%T00:00:00.000Z' AND "end" LIKE '%T00:00:00.000Z') AS day_aligned_segments,
- COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%-01T00:00:00.000Z' AND "end" LIKE '%-01T00:00:00.000Z') AS month_aligned_segments,
- COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%-01-01T00:00:00.000Z' AND "end" LIKE '%-01-01T00:00:00.000Z') AS year_aligned_segments,
- SUM("size") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS total_data_size,
- SUM("size" * "num_replicas") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS replicated_size,
- MIN("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS min_segment_rows,
- AVG("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS avg_segment_rows,
- MAX("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS max_segment_rows,
- SUM("num_rows") FILTER (WHERE (is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AS total_rows,
- CASE
- WHEN SUM("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) <> 0
- THEN (
- SUM("size") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) /
- SUM("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0)
- )
- ELSE 0
- END AS avg_row_size
+ static query(hiddenColumns: LocalStorageBackedArray<string>) {
+ const columns = compact(
+ [
+ hiddenColumns.exists('Datasource name') && `datasource`,
+ (hiddenColumns.exists('Availability') || hiddenColumns.exists('Segment granularity')) &&
+ `COUNT(*) FILTER (WHERE (is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AS num_segments`,
+ (hiddenColumns.exists('Availability') || hiddenColumns.exists('Availability detail')) && [
+ `COUNT(*) FILTER (WHERE is_published = 1 AND is_overshadowed = 0 AND is_available = 0) AS num_segments_to_load`,
+ `COUNT(*) FILTER (WHERE is_available = 1 AND NOT ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1)) AS num_segments_to_drop`,
+ ],
+ hiddenColumns.exists('Total data size') &&
+ `SUM("size") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS total_data_size`,
+ hiddenColumns.exists('Segment size') && [
+ `MIN("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS min_segment_rows`,
+ `AVG("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS avg_segment_rows`,
+ `MAX("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS max_segment_rows`,
+ ],
+ hiddenColumns.exists('Segment granularity') && [
+ `COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%:00.000Z' AND "end" LIKE '%:00.000Z') AS minute_aligned_segments`,
+ `COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%:00:00.000Z' AND "end" LIKE '%:00:00.000Z') AS hour_aligned_segments`,
+ `COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%T00:00:00.000Z' AND "end" LIKE '%T00:00:00.000Z') AS day_aligned_segments`,
+ `COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%-01T00:00:00.000Z' AND "end" LIKE '%-01T00:00:00.000Z') AS month_aligned_segments`,
+ `COUNT(*) FILTER (WHERE ((is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AND "start" LIKE '%-01-01T00:00:00.000Z' AND "end" LIKE '%-01-01T00:00:00.000Z') AS year_aligned_segments`,
+ ],
+ hiddenColumns.exists('Total rows') &&
+ `SUM("num_rows") FILTER (WHERE (is_published = 1 AND is_overshadowed = 0) OR is_realtime = 1) AS total_rows`,
+ hiddenColumns.exists('Avg. row size') &&
+ `CASE WHEN SUM("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) <> 0 THEN (SUM("size") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) / SUM("num_rows") FILTER (WHERE is_published = 1 AND is_overshadowed = 0)) ELSE 0 END AS avg_row_size`,
+ hiddenColumns.exists('Replicated size') &&
+ `SUM("size" * "num_replicas") FILTER (WHERE is_published = 1 AND is_overshadowed = 0) AS replicated_size`,
+ ].flat(),
+ );
+
+ if (!columns.length) {
+ columns.push(`datasource`);
+ }
+
+ return `SELECT
+${columns.join(',\n')}
FROM sys.segments
GROUP BY 1
ORDER BY 1`;
+ }
static formatRules(rules: Rule[]): string {
if (rules.length === 0) {
@@ -280,7 +301,7 @@ ORDER BY 1`;
}
}
- private datasourceQueryManager: QueryManager<Capabilities, DatasourcesAndDefaultRules>;
+ private datasourceQueryManager: QueryManager<DatasourceQuery, DatasourcesAndDefaultRules>;
private tiersQueryManager: QueryManager<Capabilities, string[]>;
constructor(props: DatasourcesViewProps, context: any) {
@@ -312,10 +333,16 @@ ORDER BY 1`;
};
this.datasourceQueryManager = new QueryManager({
- processQuery: async capabilities => {
+ processQuery: async (
+ { capabilities, hiddenColumns, showUnused },
+ _cancelToken,
+ setIntermediateQuery,
+ ) => {
let datasources: DatasourceQueryResultRow[];
if (capabilities.hasSql()) {
- datasources = await queryDruidSql({ query: DatasourcesView.DATASOURCE_SQL });
+ const query = DatasourcesView.query(hiddenColumns);
+ setIntermediateQuery(query);
+ datasources = await queryDruidSql({ query });
} else if (capabilities.hasCoordinatorAccess()) {
const datasourcesResp = await Api.instance.get(
'/druid/coordinator/v1/datasources?simple',
@@ -330,7 +357,6 @@ ORDER BY 1`;
const numSegments = availableSegments + segmentsToLoad;
return {
datasource: d.name,
- num_available_segments: availableSegments,
num_segments: numSegments,
num_segments_to_load: segmentsToLoad,
num_segments_to_drop: 0,
@@ -366,11 +392,9 @@ ORDER BY 1`;
const seen = countBy(datasources, x => x.datasource);
let unused: string[] = [];
- if (this.state.showUnused) {
- // Using 'includeDisabled' parameter for compatibility.
- // Should be changed to 'includeUnused' in Druid 0.17
+ if (showUnused) {
const unusedResp = await Api.instance.get(
- '/druid/coordinator/v1/metadata/datasources?includeDisabled',
+ '/druid/coordinator/v1/metadata/datasources?includeUnused',
);
unused = unusedResp.data.filter((d: string) => !seen[d]);
}
@@ -442,9 +466,15 @@ ORDER BY 1`;
this.tiersQueryManager.rerunLastQuery(auto);
};
+ private fetchDatasourceData() {
+ const { capabilities } = this.props;
+ const { hiddenColumns, showUnused } = this.state;
+ this.datasourceQueryManager.runQuery({ capabilities, hiddenColumns, showUnused });
+ }
+
componentDidMount(): void {
const { capabilities } = this.props;
- this.datasourceQueryManager.runQuery(capabilities);
+ this.fetchDatasourceData();
this.tiersQueryManager.runQuery(capabilities);
window.addEventListener('resize', this.handleResize);
}
@@ -477,7 +507,7 @@ ORDER BY 1`;
this.setState({ datasourceToMarkAsUnusedAllSegmentsIn: undefined });
}}
onSuccess={() => {
- this.datasourceQueryManager.rerunLastQuery();
+ this.fetchDatasourceData();
}}
>
<p>
@@ -510,7 +540,7 @@ ORDER BY 1`;
this.setState({ datasourceToMarkAllNonOvershadowedSegmentsAsUsedIn: undefined });
}}
onSuccess={() => {
- this.datasourceQueryManager.rerunLastQuery();
+ this.fetchDatasourceData();
}}
>
<p>{`Are you sure you want to mark as used all non-overshadowed segments in '${datasourceToMarkAllNonOvershadowedSegmentsAsUsedIn}'?`}</p>
@@ -547,7 +577,7 @@ ORDER BY 1`;
this.setState({ datasourceToMarkSegmentsByIntervalIn: undefined });
}}
onSuccess={() => {
- this.datasourceQueryManager.rerunLastQuery();
+ this.fetchDatasourceData();
}}
>
<p>{`Please select the interval in which you want to mark segments as ${usedWord} in '${datasourceToMarkSegmentsByIntervalIn}'?`}</p>
@@ -588,7 +618,7 @@ ORDER BY 1`;
this.setState({ killDatasource: undefined });
}}
onSuccess={() => {
- this.datasourceQueryManager.rerunLastQuery();
+ this.fetchDatasourceData();
}}
warningChecks={[
`I understand that this operation will delete all metadata about the unused segments of ${killDatasource} and removes them from deep storage.`,
@@ -605,6 +635,7 @@ ORDER BY 1`;
renderBulkDatasourceActions() {
const { goToQuery, capabilities } = this.props;
+ const lastDatasourcesQuery = this.datasourceQueryManager.getLastIntermediateQuery();
return (
<MoreButton
@@ -623,7 +654,11 @@ ORDER BY 1`;
<MenuItem
icon={IconNames.APPLICATION}
text="View SQL query for table"
- onClick={() => goToQuery(DatasourcesView.DATASOURCE_SQL)}
+ disabled={!lastDatasourcesQuery}
+ onClick={() => {
+ if (!lastDatasourcesQuery) return;
+ goToQuery(lastDatasourcesQuery);
+ }}
/>
)}
<MenuItem
@@ -680,7 +715,7 @@ ORDER BY 1`;
message: 'Retention rules submitted successfully',
intent: Intent.SUCCESS,
});
- this.datasourceQueryManager.rerunLastQuery();
+ this.fetchDatasourceData();
};
private editDefaultRules = () => {
@@ -705,7 +740,7 @@ ORDER BY 1`;
try {
await Api.instance.post(`/druid/coordinator/v1/config/compaction`, compactionConfig);
this.setState({ compactionDialogOpenOn: undefined });
- this.datasourceQueryManager.rerunLastQuery();
+ this.fetchDatasourceData();
} catch (e) {
AppToaster.show({
message: getDruidErrorMessage(e),
@@ -728,9 +763,7 @@ ORDER BY 1`;
await Api.instance.delete(
`/druid/coordinator/v1/config/compaction/${Api.encodePath(datasource)}`,
);
- this.setState({ compactionDialogOpenOn: undefined }, () =>
- this.datasourceQueryManager.rerunLastQuery(),
- );
+ this.setState({ compactionDialogOpenOn: undefined }, () => this.fetchDatasourceData());
} catch (e) {
AppToaster.show({
message: getDruidErrorMessage(e),
@@ -743,10 +776,10 @@ ORDER BY 1`;
};
private toggleUnused(showUnused: boolean) {
- if (!showUnused) {
- this.datasourceQueryManager.rerunLastQuery();
- }
- this.setState({ showUnused: !showUnused });
+ this.setState({ showUnused: !showUnused }, () => {
+ if (showUnused) return;
+ this.fetchDatasourceData();
+ });
}
getDatasourceActions(
@@ -979,18 +1012,11 @@ ORDER BY 1`;
{
Header: 'Availability',
show: hiddenColumns.exists('Availability'),
- id: 'availability',
filterable: false,
minWidth: 200,
- accessor: row => {
- return {
- num_available: row.num_available_segments,
- num_total: row.num_segments,
- };
- },
- Cell: ({ original }) => {
- const { datasource, num_available_segments, num_segments, unused } = original;
-
+ accessor: 'num_segments',
+ Cell: ({ value: num_segments, original }) => {
+ const { datasource, unused, num_segments_to_load } = original;
if (unused) {
return (
<span>
@@ -1005,7 +1031,9 @@ ORDER BY 1`;
{pluralIfNeeded(num_segments, 'segment')}
</a>
);
- if (num_available_segments === num_segments) {
+ if (typeof num_segments_to_load !== 'number' || typeof num_segments !== 'number') {
+ return '-';
+ } else if (num_segments_to_load === 0) {
return (
<span>
<span style={{ color: DatasourcesView.FULLY_AVAILABLE_COLOR }}>
@@ -1015,22 +1043,16 @@ ORDER BY 1`;
</span>
);
} else {
+ const numAvailableSegments = num_segments - num_segments_to_load;
const percentAvailable = (
- Math.floor((num_available_segments / num_segments) * 1000) / 10
+ Math.floor((numAvailableSegments / num_segments) * 1000) / 10
).toFixed(1);
- const missing = num_segments - num_available_segments;
- const segmentsMissingEl = (
- <a onClick={() => goToSegments(datasource, true)}>{`${pluralIfNeeded(
- missing,
- 'segment',
- )} unavailable`}</a>
- );
return (
<span>
<span style={{ color: DatasourcesView.PARTIALLY_AVAILABLE_COLOR }}>
- {num_available_segments ? '\u25cf' : '\u25cb'}
+ {numAvailableSegments ? '\u25cf' : '\u25cb'}
</span>
- {percentAvailable}% available ({segmentsEl}, {segmentsMissingEl})
+ {percentAvailable}% available ({segmentsEl})
</span>
);
}
@@ -1042,9 +1064,8 @@ ORDER BY 1`;
},
},
{
- Header: twoLines('Segment load/drop', 'queues'),
- show: hiddenColumns.exists('Segment load/drop queues'),
- id: 'load-drop',
+ Header: twoLines('Availability', 'detail'),
+ show: hiddenColumns.exists('Availability detail'),
accessor: 'num_segments_to_load',
filterable: false,
minWidth: 100,
@@ -1069,21 +1090,25 @@ ORDER BY 1`;
accessor: 'avg_segment_rows',
filterable: false,
width: 220,
- Cell: ({ value, original }) => (
- <>
- <BracedText
- text={formatSegmentRows(original.min_segment_rows)}
- braces={minSegmentRowsValues}
- />{' '}
- {' '}
- <BracedText text={formatSegmentRows(value)} braces={avgSegmentRowsValues} />{' '}
- {' '}
- <BracedText
- text={formatSegmentRows(original.max_segment_rows)}
- braces={maxSegmentRowsValues}
- />
- </>
- ),
+ Cell: ({ value, original }) => {
+ const { min_segment_rows, max_segment_rows } = original;
+ if (isNaN(value) || isNaN(min_segment_rows) || isNaN(max_segment_rows)) return '-';
+ return (
+ <>
+ <BracedText
+ text={formatSegmentRows(min_segment_rows)}
+ braces={minSegmentRowsValues}
+ />{' '}
+ {' '}
+ <BracedText text={formatSegmentRows(value)} braces={avgSegmentRowsValues} />{' '}
+ {' '}
+ <BracedText
+ text={formatSegmentRows(max_segment_rows)}
+ braces={maxSegmentRowsValues}
+ />
+ </>
+ );
+ },
},
{
Header: twoLines('Segment', 'granularity'),
@@ -1093,24 +1118,32 @@ ORDER BY 1`;
filterable: false,
width: 100,
Cell: ({ original }) => {
+ const {
+ num_segments,
+ minute_aligned_segments,
+ hour_aligned_segments,
+ day_aligned_segments,
+ month_aligned_segments,
+ year_aligned_segments,
+ } = original;
const segmentGranularities: string[] = [];
- if (!original.num_segments) return '-';
- if (original.num_segments - original.minute_aligned_segments) {
+ if (!num_segments || isNaN(year_aligned_segments)) return '-';
+ if (num_segments - minute_aligned_segments) {
segmentGranularities.push('Sub minute');
}
- if (original.minute_aligned_segments - original.hour_aligned_segments) {
+ if (minute_aligned_segments - hour_aligned_segments) {
segmentGranularities.push('Minute');
}
- if (original.hour_aligned_segments - original.day_aligned_segments) {
+ if (hour_aligned_segments - day_aligned_segments) {
segmentGranularities.push('Hour');
}
- if (original.day_aligned_segments - original.month_aligned_segments) {
+ if (day_aligned_segments - month_aligned_segments) {
segmentGranularities.push('Day');
}
- if (original.month_aligned_segments - original.year_aligned_segments) {
+ if (month_aligned_segments - year_aligned_segments) {
segmentGranularities.push('Month');
}
- if (original.year_aligned_segments) {
+ if (year_aligned_segments) {
segmentGranularities.push('Year');
}
return segmentGranularities.join(', ');
@@ -1122,9 +1155,10 @@ ORDER BY 1`;
accessor: 'total_rows',
filterable: false,
width: 100,
- Cell: ({ value }) => (
- <BracedText text={formatTotalRows(value)} braces={totalRowsValues} />
- ),
+ Cell: ({ value }) => {
+ if (isNaN(value)) return '-';
+ return <BracedText text={formatTotalRows(value)} braces={totalRowsValues} />;
+ },
},
{
Header: twoLines('Avg. row size', '(bytes)'),
@@ -1132,9 +1166,10 @@ ORDER BY 1`;
accessor: 'avg_row_size',
filterable: false,
width: 100,
- Cell: ({ value }) => (
- <BracedText text={formatAvgRowSize(value)} braces={avgRowSizeValues} />
- ),
+ Cell: ({ value }) => {
+ if (isNaN(value)) return '-';
+ return <BracedText text={formatAvgRowSize(value)} braces={avgRowSizeValues} />;
+ },
},
{
Header: twoLines('Replicated', 'size'),
@@ -1142,9 +1177,12 @@ ORDER BY 1`;
accessor: 'replicated_size',
filterable: false,
width: 100,
- Cell: ({ value }) => (
- <BracedText text={formatReplicatedSize(value)} braces={replicatedSizeValues} />
- ),
+ Cell: ({ value }) => {
+ if (isNaN(value)) return '-';
+ return (
+ <BracedText text={formatReplicatedSize(value)} braces={replicatedSizeValues} />
+ );
+ },
},
{
Header: 'Compaction',
@@ -1371,6 +1409,10 @@ ORDER BY 1`;
hiddenColumns: prevState.hiddenColumns.toggle(column),
}))
}
+ onClose={added => {
+ if (!added) return;
+ this.fetchDatasourceData();
+ }}
tableColumnsHidden={hiddenColumns.storedArray}
/>
</ViewControlBar>
diff --git a/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap b/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap
index 80d1467..7c3a51b 100755
--- a/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap
+++ b/web-console/src/views/segments-view/__snapshots__/segments-view.spec.tsx.snap
@@ -62,6 +62,7 @@ exports[`segments-view matches snapshot 1`] = `
]
}
onChange={[Function]}
+ onClose={[Function]}
tableColumnsHidden={Array []}
/>
</Memo(ViewControlBar)>
@@ -125,7 +126,9 @@ exports[`segments-view matches snapshot 1`] = `
Object {
"Header": "Segment ID",
"accessor": "segment_id",
+ "filterable": true,
"show": true,
+ "sortable": true,
"width": 300,
},
Object {
@@ -139,7 +142,9 @@ exports[`segments-view matches snapshot 1`] = `
"Header": "Interval",
"accessor": "interval",
"defaultSortDesc": true,
+ "filterable": true,
"show": false,
+ "sortable": true,
"width": 120,
},
Object {
@@ -147,7 +152,9 @@ exports[`segments-view matches snapshot 1`] = `
"Header": "Start",
"accessor": "start",
"defaultSortDesc": true,
+ "filterable": true,
"show": true,
+ "sortable": true,
"width": 120,
},
Object {
@@ -155,14 +162,18 @@ exports[`segments-view matches snapshot 1`] = `
"Header": "End",
"accessor": "end",
"defaultSortDesc": true,
+ "filterable": true,
"show": true,
+ "sortable": true,
"width": 120,
},
Object {
"Header": "Version",
"accessor": "version",
"defaultSortDesc": true,
+ "filterable": true,
"show": true,
+ "sortable": true,
"width": 120,
},
Object {
@@ -171,6 +182,7 @@ exports[`segments-view matches snapshot 1`] = `
"accessor": "time_span",
"filterable": true,
"show": true,
+ "sortable": true,
"width": 100,
},
Object {
@@ -179,6 +191,7 @@ exports[`segments-view matches snapshot 1`] = `
"accessor": "partitioning",
"filterable": true,
"show": true,
+ "sortable": true,
"width": 100,
},
Object {
@@ -186,6 +199,7 @@ exports[`segments-view matches snapshot 1`] = `
"accessor": "partition_num",
"filterable": false,
"show": true,
+ "sortable": true,
"width": 60,
},
Object {
@@ -195,6 +209,7 @@ exports[`segments-view matches snapshot 1`] = `
"defaultSortDesc": true,
"filterable": false,
"show": true,
+ "sortable": true,
},
Object {
"Cell": [Function],
diff --git a/web-console/src/views/segments-view/segments-view.tsx b/web-console/src/views/segments-view/segments-view.tsx
index 0df4da6..979d3f6 100644
--- a/web-console/src/views/segments-view/segments-view.tsx
+++ b/web-console/src/views/segments-view/segments-view.tsx
@@ -19,7 +19,6 @@
import { Button, ButtonGroup, Intent, Label, MenuItem } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import { SqlExpression, SqlRef } from 'druid-query-toolkit';
-import * as JSONBig from 'json-bigint-native';
import React from 'react';
import ReactTable, { Filter } from 'react-table';
@@ -39,11 +38,13 @@ import { SegmentTableActionDialog } from '../../dialogs/segments-table-action-di
import { Api } from '../../singletons';
import {
addFilter,
+ booleanCustomTableFilter,
compact,
deepGet,
filterMap,
formatBytes,
formatInteger,
+ getNeedleAndMode,
LocalStorageKeys,
makeBooleanFilter,
queryDruidSql,
@@ -124,6 +125,8 @@ interface TableState {
}
interface SegmentsQuery extends TableState {
+ hiddenColumns: LocalStorageBackedArray<string>;
+ capabilities: Capabilities;
groupByInterval: boolean;
}
@@ -131,6 +134,7 @@ interface SegmentQueryResultRow {
datasource: string;
start: string;
end: string;
+ interval: string;
segment_id: string;
version: string;
time_span: string;
@@ -147,7 +151,6 @@ interface SegmentQueryResultRow {
export interface SegmentsViewState {
segmentsState: QueryState<SegmentQueryResultRow[]>;
- trimmedSegments?: SegmentQueryResultRow[];
segmentFilter: Filter[];
segmentTableActionDialogId?: string;
datasourceTableActionDialogId?: string;
@@ -161,33 +164,74 @@ export interface SegmentsViewState {
export class SegmentsView extends React.PureComponent<SegmentsViewProps, SegmentsViewState> {
static PAGE_SIZE = 25;
- static WITH_QUERY = `WITH s AS (
- SELECT
- "segment_id", "datasource", "start", "end", "size", "version",
- CASE
- WHEN "start" LIKE '%-01-01T00:00:00.000Z' AND "end" LIKE '%-01-01T00:00:00.000Z' THEN 'Year'
- WHEN "start" LIKE '%-01T00:00:00.000Z' AND "end" LIKE '%-01T00:00:00.000Z' THEN 'Month'
- WHEN "start" LIKE '%T00:00:00.000Z' AND "end" LIKE '%T00:00:00.000Z' THEN 'Day'
- WHEN "start" LIKE '%:00:00.000Z' AND "end" LIKE '%:00:00.000Z' THEN 'Hour'
- WHEN "start" LIKE '%:00.000Z' AND "end" LIKE '%:00.000Z' THEN 'Minute'
- ELSE 'Sub minute'
- END AS "time_span",
- CASE
- WHEN "shard_spec" LIKE '%"type":"numbered"%' THEN 'dynamic'
- WHEN "shard_spec" LIKE '%"type":"hashed"%' THEN 'hashed'
- WHEN "shard_spec" LIKE '%"type":"single"%' THEN 'single_dim'
- WHEN "shard_spec" LIKE '%"type":"none"%' THEN 'none'
- WHEN "shard_spec" LIKE '%"type":"linear"%' THEN 'linear'
- WHEN "shard_spec" LIKE '%"type":"numbered_overwrite"%' THEN 'numbered_overwrite'
- ELSE '-'
- END AS "partitioning",
- "partition_num", "num_replicas", "num_rows",
- "is_published", "is_available", "is_realtime", "is_overshadowed"
- FROM sys.segments
-)`;
-
- private segmentsSqlQueryManager: QueryManager<SegmentsQuery, SegmentQueryResultRow[]>;
- private segmentsNoSqlQueryManager: QueryManager<null, SegmentQueryResultRow[]>;
+ static baseQuery(hiddenColumns: LocalStorageBackedArray<string>) {
+ const columns = compact([
+ hiddenColumns.exists('Segment ID') && `"segment_id"`,
+ hiddenColumns.exists('Datasource') && `"datasource"`,
+ hiddenColumns.exists('Start') && `"start"`,
+ hiddenColumns.exists('End') && `"end"`,
+ hiddenColumns.exists('Version') && `"version"`,
+ hiddenColumns.exists('Time span') &&
+ `CASE
+ WHEN "start" LIKE '%-01-01T00:00:00.000Z' AND "end" LIKE '%-01-01T00:00:00.000Z' THEN 'Year'
+ WHEN "start" LIKE '%-01T00:00:00.000Z' AND "end" LIKE '%-01T00:00:00.000Z' THEN 'Month'
+ WHEN "start" LIKE '%T00:00:00.000Z' AND "end" LIKE '%T00:00:00.000Z' THEN 'Day'
+ WHEN "start" LIKE '%:00:00.000Z' AND "end" LIKE '%:00:00.000Z' THEN 'Hour'
+ WHEN "start" LIKE '%:00.000Z' AND "end" LIKE '%:00.000Z' THEN 'Minute'
+ ELSE 'Sub minute'
+END AS "time_span"`,
+ hiddenColumns.exists('Partitioning') &&
+ `CASE
+ WHEN "shard_spec" LIKE '%"type":"numbered"%' THEN 'dynamic'
+ WHEN "shard_spec" LIKE '%"type":"hashed"%' THEN 'hashed'
+ WHEN "shard_spec" LIKE '%"type":"single"%' THEN 'single_dim'
+ WHEN "shard_spec" LIKE '%"type":"none"%' THEN 'none'
+ WHEN "shard_spec" LIKE '%"type":"linear"%' THEN 'linear'
+ WHEN "shard_spec" LIKE '%"type":"numbered_overwrite"%' THEN 'numbered_overwrite'
+ ELSE '-'
+END AS "partitioning"`,
+ hiddenColumns.exists('Partition') && `"partition_num"`,
+ hiddenColumns.exists('Size') && `"size"`,
+ hiddenColumns.exists('Num rows') && `"num_rows"`,
+ hiddenColumns.exists('Replicas') && `"num_replicas"`,
+ hiddenColumns.exists('Is published') && `"is_published"`,
+ hiddenColumns.exists('Is available') && `"is_available"`,
+ hiddenColumns.exists('Is realtime') && `"is_realtime"`,
+ hiddenColumns.exists('Is overshadowed') && `"is_overshadowed"`,
+ ]);
+
+ if (!columns.length) {
+ columns.push(`"segment_id"`);
+ }
+
+ return `WITH s AS (SELECT\n${columns.join(',\n')}\nFROM sys.segments)`;
+ }
+
+ static computeTimeSpan(start: string, end: string): string {
+ if (start.endsWith('-01-01T00:00:00.000Z') && end.endsWith('-01-01T00:00:00.000Z')) {
+ return 'Year';
+ }
+
+ if (start.endsWith('-01T00:00:00.000Z') && end.endsWith('-01T00:00:00.000Z')) {
+ return 'Month';
+ }
+
+ if (start.endsWith('T00:00:00.000Z') && end.endsWith('T00:00:00.000Z')) {
+ return 'Day';
+ }
+
+ if (start.endsWith(':00:00.000Z') && end.endsWith(':00:00.000Z')) {
+ return 'Hour';
+ }
+
+ if (start.endsWith(':00.000Z') && end.endsWith(':00.000Z')) {
+ return 'Minute';
+ }
+
+ return 'Sub minute';
+ }
+
+ private segmentsQueryManager: QueryManager<SegmentsQuery, SegmentQueryResultRow[]>;
private lastTableState: TableState | undefined;
@@ -208,194 +252,189 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
groupByInterval: false,
};
- this.segmentsSqlQueryManager = new QueryManager({
+ this.segmentsQueryManager = new QueryManager({
debounceIdle: 500,
processQuery: async (query: SegmentsQuery, _cancelToken, setIntermediateQuery) => {
- const whereParts = filterMap(query.filtered, (f: Filter) => {
- if (f.id.startsWith('is_')) {
- if (f.value === 'all') return;
- return SqlRef.columnWithQuotes(f.id).equal(f.value === 'true' ? 1 : 0);
- } else {
- return sqlQueryCustomTableFilter(f);
+ const {
+ page,
+ pageSize,
+ filtered,
+ sorted,
+ hiddenColumns,
+ capabilities,
+ groupByInterval,
+ } = query;
+
+ if (capabilities.hasSql()) {
+ const whereParts = filterMap(filtered, (f: Filter) => {
+ if (f.id.startsWith('is_')) {
+ if (f.value === 'all') return;
+ return SqlRef.columnWithQuotes(f.id).equal(f.value === 'true' ? 1 : 0);
+ } else {
+ return sqlQueryCustomTableFilter(f);
+ }
+ });
+
+ let queryParts: string[];
+
+ let whereClause = '';
+ if (whereParts.length) {
+ whereClause = SqlExpression.and(...whereParts).toString();
}
- });
- let queryParts: string[];
+ if (groupByInterval) {
+ const innerQuery = compact([
+ `SELECT "start" || '/' || "end" AS "interval"`,
+ `FROM sys.segments`,
+ whereClause ? `WHERE ${whereClause}` : undefined,
+ `GROUP BY 1`,
+ `ORDER BY 1 DESC`,
+ `LIMIT ${pageSize}`,
+ page ? `OFFSET ${page * pageSize}` : undefined,
+ ]).join('\n');
+
+ const intervals: string = (await queryDruidSql({ query: innerQuery }))
+ .map(row => `'${row.interval}'`)
+ .join(', ');
+
+ queryParts = compact([
+ SegmentsView.baseQuery(hiddenColumns),
+ `SELECT "start" || '/' || "end" AS "interval", *`,
+ `FROM s`,
+ `WHERE`,
+ intervals ? ` ("start" || '/' || "end") IN (${intervals})` : 'FALSE',
+ whereClause ? ` AND ${whereClause}` : '',
+ ]);
+
+ if (sorted.length) {
+ queryParts.push(
+ 'ORDER BY ' +
+ sorted
+ .map((sort: any) => `${SqlRef.column(sort.id)} ${sort.desc ? 'DESC' : 'ASC'}`)
+ .join(', '),
+ );
+ }
- let whereClause = '';
- if (whereParts.length) {
- whereClause = SqlExpression.and(...whereParts).toString();
- }
+ queryParts.push(`LIMIT ${pageSize * 1000}`);
+ } else {
+ queryParts = [SegmentsView.baseQuery(hiddenColumns), `SELECT *`, `FROM s`];
- if (query.groupByInterval) {
- const innerQuery = compact([
- `SELECT "start" || '/' || "end" AS "interval"`,
- `FROM sys.segments`,
- whereClause ? `WHERE ${whereClause}` : undefined,
- `GROUP BY 1`,
- `ORDER BY 1 DESC`,
- `LIMIT ${query.pageSize}`,
- query.page ? `OFFSET ${query.page * query.pageSize}` : undefined,
- ]).join('\n');
-
- const intervals: string = (await queryDruidSql({ query: innerQuery }))
- .map(row => `'${row.interval}'`)
- .join(', ');
-
- queryParts = compact([
- SegmentsView.WITH_QUERY,
- `SELECT "start" || '/' || "end" AS "interval", *`,
- `FROM s`,
- `WHERE`,
- intervals ? ` ("start" || '/' || "end") IN (${intervals})` : 'FALSE',
- whereClause ? ` AND ${whereClause}` : '',
- ]);
-
- if (query.sorted.length) {
- queryParts.push(
- 'ORDER BY ' +
- query.sorted
- .map((sort: any) => `${JSONBig.stringify(sort.id)} ${sort.desc ? 'DESC' : 'ASC'}`)
- .join(', '),
- );
- }
+ if (whereClause) {
+ queryParts.push(`WHERE ${whereClause}`);
+ }
- queryParts.push(`LIMIT ${query.pageSize * 1000}`);
- } else {
- queryParts = [SegmentsView.WITH_QUERY, `SELECT *`, `FROM s`];
+ if (sorted.length) {
+ queryParts.push(
+ 'ORDER BY ' +
+ sorted
+ .map((sort: any) => `${SqlRef.column(sort.id)} ${sort.desc ? 'DESC' : 'ASC'}`)
+ .join(', '),
+ );
+ }
- if (whereClause) {
- queryParts.push(`WHERE ${whereClause}`);
- }
+ queryParts.push(`LIMIT ${pageSize}`);
- if (query.sorted.length) {
- queryParts.push(
- 'ORDER BY ' +
- query.sorted
- .map((sort: any) => `${JSONBig.stringify(sort.id)} ${sort.desc ? 'DESC' : 'ASC'}`)
- .join(', '),
+ if (page) {
+ queryParts.push(`OFFSET ${page * pageSize}`);
+ }
+ }
+ const sqlQuery = queryParts.join('\n');
+ setIntermediateQuery(sqlQuery);
+ return await queryDruidSql({ query: sqlQuery });
+ } else if (capabilities.hasCoordinatorAccess()) {
+ let datasourceList: string[] = (await Api.instance.get(
+ '/druid/coordinator/v1/metadata/datasources',
+ )).data;
+
+ const datasourceFilter = filtered.find(({ id }) => id === 'datasource');
+ if (datasourceFilter) {
+ datasourceList = datasourceList.filter(datasource =>
+ booleanCustomTableFilter(datasourceFilter, datasource),
);
}
- queryParts.push(`LIMIT ${query.pageSize}`);
-
- if (query.page) {
- queryParts.push(`OFFSET ${query.page * query.pageSize}`);
+ if (sorted.length && sorted[0].id === 'datasource') {
+ datasourceList.sort(
+ sorted[0].desc ? (d1, d2) => d1.localeCompare(d2) : (d1, d2) => d2.localeCompare(d1),
+ );
}
- }
- const sqlQuery = queryParts.join('\n');
- setIntermediateQuery(sqlQuery);
- return await queryDruidSql({ query: sqlQuery });
- },
- onStateChange: segmentsState => {
- this.setState({
- segmentsState,
- });
- },
- });
- this.segmentsNoSqlQueryManager = new QueryManager({
- processQuery: async () => {
- const datasourceList = (await Api.instance.get(
- '/druid/coordinator/v1/metadata/datasources',
- )).data;
- const nestedResults: SegmentQueryResultRow[][] = await Promise.all(
- datasourceList.map(async (d: string) => {
+ const maxResults = (page + 1) * pageSize;
+ let results: SegmentQueryResultRow[] = [];
+
+ const n = Math.min(datasourceList.length, maxResults);
+ for (let i = 0; i < n && results.length < maxResults; i++) {
const segments = (await Api.instance.get(
- `/druid/coordinator/v1/datasources/${Api.encodePath(d)}?full`,
+ `/druid/coordinator/v1/datasources/${Api.encodePath(datasourceList[i])}?full`,
)).data.segments;
+ if (!Array.isArray(segments)) continue;
+
+ let segmentQueryResultRows: SegmentQueryResultRow[] = segments.map((segment: any) => {
+ const [start, end] = segment.interval.split('/');
+ return {
+ segment_id: segment.identifier,
+ datasource: segment.dataSource,
+ start,
+ end,
+ interval: segment.interval,
+ version: segment.version,
+ time_span: SegmentsView.computeTimeSpan(start, end),
+ partitioning: deepGet(segment, 'shardSpec.type') || '-',
+ partition_num: deepGet(segment, 'shardSpec.partitionNum') || 0,
+ size: segment.size,
+ num_rows: -1,
+ num_replicas: -1,
+ is_available: -1,
+ is_published: -1,
+ is_realtime: -1,
+ is_overshadowed: -1,
+ };
+ });
- return segments.map(
- (segment: any): SegmentQueryResultRow => {
- return {
- segment_id: segment.identifier,
- datasource: segment.dataSource,
- start: segment.interval.split('/')[0],
- end: segment.interval.split('/')[1],
- version: segment.version,
- time_span: '-',
- partitioning: '-',
- partition_num: deepGet(segment, 'shardSpec.partitionNum') || 0,
- size: segment.size,
- num_rows: -1,
- num_replicas: -1,
- is_available: -1,
- is_published: -1,
- is_realtime: -1,
- is_overshadowed: -1,
- };
- },
- );
- }),
- );
+ if (filtered.length) {
+ segmentQueryResultRows = segmentQueryResultRows.filter((d: SegmentQueryResultRow) => {
+ return filtered.every(filter => {
+ return booleanCustomTableFilter(
+ filter,
+ d[filter.id as keyof SegmentQueryResultRow],
+ );
+ });
+ });
+ }
- return nestedResults.flat().sort((d1, d2) => {
- return d2.start.localeCompare(d1.start);
- });
+ results = results.concat(segmentQueryResultRows);
+ }
+
+ return results.slice(page * pageSize, maxResults);
+ } else {
+ throw new Error('must have SQL or coordinator access to load this view');
+ }
},
onStateChange: segmentsState => {
this.setState({
- trimmedSegments: segmentsState.data
- ? segmentsState.data.slice(0, SegmentsView.PAGE_SIZE)
- : undefined,
segmentsState,
});
},
});
}
- componentDidMount(): void {
- const { capabilities } = this.props;
- if (!capabilities.hasSql() && capabilities.hasCoordinatorAccess()) {
- this.segmentsNoSqlQueryManager.runQuery(null);
- }
- }
-
componentWillUnmount(): void {
- this.segmentsSqlQueryManager.terminate();
- this.segmentsNoSqlQueryManager.terminate();
+ this.segmentsQueryManager.terminate();
}
private fetchData = (groupByInterval: boolean, tableState?: TableState) => {
+ const { capabilities } = this.props;
+ const { hiddenColumns } = this.state;
if (tableState) this.lastTableState = tableState;
const { page, pageSize, filtered, sorted } = this.lastTableState!;
- this.segmentsSqlQueryManager.runQuery({
+ this.segmentsQueryManager.runQuery({
page,
pageSize,
filtered,
sorted,
- groupByInterval: groupByInterval,
- });
- };
-
- private fetchClientSideData = (tableState?: TableState) => {
- if (tableState) this.lastTableState = tableState;
- const { page, pageSize, filtered, sorted } = this.lastTableState!;
-
- this.setState(state => {
- const allSegments = state.segmentsState.data;
- if (!allSegments) return {};
- const sortKey = sorted[0].id as keyof SegmentQueryResultRow;
- const sortDesc = sorted[0].desc;
-
- return {
- trimmedSegments: allSegments
- .filter(d => {
- return filtered.every((f: any) => {
- return String(d[f.id as keyof SegmentQueryResultRow]).includes(f.value);
- });
- })
- .sort((d1, d2) => {
- const v1 = d1[sortKey] as any;
- const v2 = d2[sortKey] as any;
- if (typeof v1 === 'string') {
- return sortDesc ? v2.localeCompare(v1) : v1.localeCompare(v2);
- } else {
- return sortDesc ? v2 - v1 : v1 - v2;
- }
- })
- .slice(page * pageSize, (page + 1) * pageSize),
- };
+ hiddenColumns,
+ capabilities,
+ groupByInterval,
});
};
@@ -411,16 +450,10 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
}
renderSegmentsTable() {
- const {
- segmentsState,
- trimmedSegments,
- segmentFilter,
- hiddenColumns,
- groupByInterval,
- } = this.state;
+ const { segmentsState, segmentFilter, hiddenColumns, groupByInterval } = this.state;
const { capabilities } = this.props;
- const segments = trimmedSegments || segmentsState.data || [];
+ const segments = segmentsState.data || [];
const sizeValues = segments.map(d => formatBytes(d.size)).concat('(realtime)');
@@ -443,6 +476,15 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
};
};
+ const hasSql = capabilities.hasSql();
+
+ // Only allow filtering of columns other than datasource if in SQL mode or we are filtering on an exact datasource
+ const allowGeneralFilter =
+ hasSql ||
+ segmentFilter.some(
+ filter => filter.id === 'datasource' && getNeedleAndMode(filter).mode === 'exact',
+ );
+
return (
<ReactTable
data={segments}
@@ -452,16 +494,12 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
manual
filterable
filtered={segmentFilter}
- defaultSorted={[{ id: 'start', desc: true }]}
+ defaultSorted={[hasSql ? { id: 'start', desc: true } : { id: 'datasource', desc: false }]}
onFilteredChange={filtered => {
this.setState({ segmentFilter: filtered });
}}
onFetchData={tableState => {
- if (capabilities.hasSql()) {
- this.fetchData(groupByInterval, tableState);
- } else if (capabilities.hasCoordinatorAccess()) {
- this.fetchClientSideData(tableState);
- }
+ this.fetchData(groupByInterval, tableState);
}}
showPageJump={false}
ofText=""
@@ -472,6 +510,8 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
show: hiddenColumns.exists('Segment ID'),
accessor: 'segment_id',
width: 300,
+ sortable: hasSql,
+ filterable: allowGeneralFilter,
},
{
Header: 'Datasource',
@@ -484,7 +524,9 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
show: groupByInterval,
accessor: 'interval',
width: 120,
+ sortable: hasSql,
defaultSortDesc: true,
+ filterable: allowGeneralFilter,
Cell: renderFilterableCell('interval'),
},
{
@@ -492,38 +534,46 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
show: hiddenColumns.exists('Start'),
accessor: 'start',
width: 120,
+ sortable: hasSql,
defaultSortDesc: true,
+ filterable: allowGeneralFilter,
Cell: renderFilterableCell('start'),
},
{
Header: 'End',
show: hiddenColumns.exists('End'),
accessor: 'end',
- defaultSortDesc: true,
width: 120,
+ sortable: hasSql,
+ defaultSortDesc: true,
+ filterable: allowGeneralFilter,
Cell: renderFilterableCell('end'),
},
{
Header: 'Version',
show: hiddenColumns.exists('Version'),
accessor: 'version',
- defaultSortDesc: true,
width: 120,
+ sortable: hasSql,
+ defaultSortDesc: true,
+ filterable: allowGeneralFilter,
},
{
Header: 'Time span',
- show: capabilities.hasSql() && hiddenColumns.exists('Time span'),
+ show: hiddenColumns.exists('Time span'),
accessor: 'time_span',
width: 100,
- filterable: true,
+ sortable: hasSql,
+ filterable: allowGeneralFilter,
Cell: renderFilterableCell('time_span'),
},
{
Header: 'Partitioning',
- show: capabilities.hasSql() && hiddenColumns.exists('Partitioning'),
+ show: hiddenColumns.exists('Partitioning'),
accessor: 'partitioning',
width: 100,
- filterable: true,
+ sortable: hasSql,
+ filterable: allowGeneralFilter,
Cell: renderFilterableCell('partitioning'),
},
{
@@ -532,12 +582,14 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
accessor: 'partition_num',
width: 60,
filterable: false,
+ sortable: hasSql,
},
{
Header: 'Size',
show: hiddenColumns.exists('Size'),
accessor: 'size',
filterable: false,
+ sortable: hasSql,
defaultSortDesc: true,
Cell: row => (
<BracedText
@@ -552,7 +604,7 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
},
{
Header: 'Num rows',
- show: capabilities.hasSql() && hiddenColumns.exists('Num rows'),
+ show: hasSql && hiddenColumns.exists('Num rows'),
accessor: 'num_rows',
filterable: false,
defaultSortDesc: true,
@@ -565,7 +617,7 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
},
{
Header: 'Replicas',
- show: capabilities.hasSql() && hiddenColumns.exists('Replicas'),
+ show: hasSql && hiddenColumns.exists('Replicas'),
accessor: 'num_replicas',
width: 60,
filterable: false,
@@ -573,28 +625,28 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
},
{
Header: 'Is published',
- show: capabilities.hasSql() && hiddenColumns.exists('Is published'),
+ show: hasSql && hiddenColumns.exists('Is published'),
id: 'is_published',
accessor: row => String(Boolean(row.is_published)),
Filter: makeBooleanFilter(),
},
{
Header: 'Is realtime',
- show: capabilities.hasSql() && hiddenColumns.exists('Is realtime'),
+ show: hasSql && hiddenColumns.exists('Is realtime'),
id: 'is_realtime',
accessor: row => String(Boolean(row.is_realtime)),
Filter: makeBooleanFilter(),
},
{
Header: 'Is available',
- show: capabilities.hasSql() && hiddenColumns.exists('Is available'),
+ show: hasSql && hiddenColumns.exists('Is available'),
id: 'is_available',
accessor: row => String(Boolean(row.is_available)),
Filter: makeBooleanFilter(),
},
{
Header: 'Is overshadowed',
- show: capabilities.hasSql() && hiddenColumns.exists('Is overshadowed'),
+ show: hasSql && hiddenColumns.exists('Is overshadowed'),
id: 'is_overshadowed',
accessor: row => String(Boolean(row.is_overshadowed)),
Filter: makeBooleanFilter(),
@@ -654,8 +706,7 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
this.setState({ terminateSegmentId: undefined });
}}
onSuccess={() => {
- this.segmentsNoSqlQueryManager.rerunLastQuery();
- this.segmentsSqlQueryManager.rerunLastQuery();
+ this.segmentsQueryManager.rerunLastQuery();
}}
>
<p>{`Are you sure you want to drop segment '${terminateSegmentId}'?`}</p>
@@ -666,7 +717,7 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
renderBulkSegmentsActions() {
const { goToQuery, capabilities } = this.props;
- const lastSegmentsQuery = this.segmentsSqlQueryManager.getLastIntermediateQuery();
+ const lastSegmentsQuery = this.segmentsQueryManager.getLastIntermediateQuery();
return (
<MoreButton>
@@ -700,11 +751,7 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
<div className="segments-view app-view">
<ViewControlBar label="Segments">
<RefreshButton
- onRefresh={auto =>
- capabilities.hasSql()
- ? this.segmentsSqlQueryManager.rerunLastQuery(auto)
- : this.segmentsNoSqlQueryManager.rerunLastQuery(auto)
- }
+ onRefresh={auto => this.segmentsQueryManager.rerunLastQuery(auto)}
localStorageKey={LocalStorageKeys.SEGMENTS_REFRESH_RATE}
/>
<Label>Group by</Label>
@@ -713,11 +760,7 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
active={!groupByInterval}
onClick={() => {
this.setState({ groupByInterval: false });
- if (capabilities.hasSql()) {
- this.fetchData(false);
- } else {
- this.fetchClientSideData();
- }
+ this.fetchData(false);
}}
>
None
@@ -740,6 +783,10 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
hiddenColumns: prevState.hiddenColumns.toggle(column),
}))
}
+ onClose={added => {
+ if (!added) return;
+ this.fetchData(groupByInterval);
+ }}
tableColumnsHidden={hiddenColumns.storedArray}
/>
</ViewControlBar>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org