You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by vo...@apache.org on 2024/04/22 22:24:50 UTC
(druid) branch better_supervisor_view created (now 06ef03402b6)
This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a change to branch better_supervisor_view
in repository https://gitbox.apache.org/repos/asf/druid.git
at 06ef03402b6 add recent errors
This branch includes the following new commits:
new 151a2060981 add rate and stats
new 78ad36ba850 better tabs
new bab61e1fd4f detail
new 06ef03402b6 add recent errors
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org
(druid) 02/04: better tabs
Posted by vo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch better_supervisor_view
in repository https://gitbox.apache.org/repos/asf/druid.git
commit 78ad36ba85028d12372fe684e178593c00235d6a
Author: Vadim Ogievetsky <va...@ogievetsky.com>
AuthorDate: Fri Apr 19 12:52:06 2024 -0700
better tabs
---
.../compaction-history-dialog.tsx | 4 +-
.../kill-datasource-dialog.tsx | 7 +-
.../supervisor-table-action-dialog.tsx | 14 +-
.../task-table-action-dialog.tsx | 40 ++---
.../supervisor-status/supervisor-status.ts | 11 +-
.../src/views/load-data-view/info-messages.tsx | 6 +-
.../src/views/load-data-view/load-data-view.tsx | 10 +-
.../src/views/segments-view/segments-view.tsx | 6 +-
.../views/supervisors-view/supervisors-view.scss | 4 +
.../views/supervisors-view/supervisors-view.tsx | 171 ++++++++++++++-------
web-console/src/views/tasks-view/tasks-view.tsx | 6 +-
11 files changed, 179 insertions(+), 100 deletions(-)
diff --git a/web-console/src/dialogs/compaction-history-dialog/compaction-history-dialog.tsx b/web-console/src/dialogs/compaction-history-dialog/compaction-history-dialog.tsx
index 4cdc916ee74..cb886d0483d 100644
--- a/web-console/src/dialogs/compaction-history-dialog/compaction-history-dialog.tsx
+++ b/web-console/src/dialogs/compaction-history-dialog/compaction-history-dialog.tsx
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-import { Button, Callout, Classes, Code, Dialog, Tab, Tabs } from '@blueprintjs/core';
+import { Button, Callout, Classes, Dialog, Tab, Tabs, Tag } from '@blueprintjs/core';
import * as JSONBig from 'json-bigint-native';
import React, { useState } from 'react';
@@ -117,7 +117,7 @@ export const CompactionHistoryDialog = React.memo(function CompactionHistoryDial
</Tabs>
) : (
<div>
- There is no compaction history for <Code>{datasource}</Code>.
+ There is no compaction history for <Tag minimal>{datasource}</Tag>.
</div>
)
) : historyState.loading ? (
diff --git a/web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx b/web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx
index f95a5a5d3b8..dba85268d00 100644
--- a/web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx
+++ b/web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-import { Code, Intent } from '@blueprintjs/core';
+import { Intent, Tag } from '@blueprintjs/core';
import React, { useState } from 'react';
import { FormGroupWithInfo, PopoverText } from '../../components';
@@ -74,13 +74,14 @@ export const KillDatasourceDialog = function KillDatasourceDialog(
warningChecks={[
<>
I understand that this operation will delete all metadata about the unused segments of{' '}
- <Code>{datasource}</Code> and removes them from deep storage.
+ <Tag minimal>{datasource}</Tag> and removes them from deep storage.
</>,
'I understand that this operation cannot be undone.',
]}
>
<p>
- Are you sure you want to permanently delete unused segments in <Code>{datasource}</Code>?
+ Are you sure you want to permanently delete unused segments in{' '}
+ <Tag minimal>{datasource}</Tag>?
</p>
<p>This action is not reversible and the data deleted will be lost.</p>
<FormGroupWithInfo
diff --git a/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-table-action-dialog.tsx b/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-table-action-dialog.tsx
index 02d9e3c28b8..5e3d9e50028 100644
--- a/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-table-action-dialog.tsx
+++ b/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-table-action-dialog.tsx
@@ -28,6 +28,8 @@ import { TableActionDialog } from '../table-action-dialog/table-action-dialog';
import { SupervisorStatisticsTable } from './supervisor-statistics-table/supervisor-statistics-table';
+type SupervisorTableActionDialogTab = 'status' | 'stats' | 'spec' | 'history';
+
interface SupervisorTableActionDialogProps {
supervisorId: string;
actions: BasicAction[];
@@ -38,7 +40,7 @@ export const SupervisorTableActionDialog = React.memo(function SupervisorTableAc
props: SupervisorTableActionDialogProps,
) {
const { supervisorId, actions, onClose } = props;
- const [activeTab, setActiveTab] = useState('status');
+ const [activeTab, setActiveTab] = useState<SupervisorTableActionDialogTab>('status');
const supervisorTableSideButtonMetadata: SideButtonMetaData[] = [
{
@@ -49,15 +51,15 @@ export const SupervisorTableActionDialog = React.memo(function SupervisorTableAc
},
{
icon: 'chart',
- text: 'Statistics',
+ text: 'Task stats',
active: activeTab === 'stats',
onClick: () => setActiveTab('stats'),
},
{
icon: 'align-left',
- text: 'Payload',
- active: activeTab === 'payload',
- onClick: () => setActiveTab('payload'),
+ text: 'Spec',
+ active: activeTab === 'spec',
+ onClick: () => setActiveTab('spec'),
},
{
icon: 'history',
@@ -88,7 +90,7 @@ export const SupervisorTableActionDialog = React.memo(function SupervisorTableAc
downloadFilename={`supervisor-stats-${supervisorId}.json`}
/>
)}
- {activeTab === 'payload' && (
+ {activeTab === 'spec' && (
<ShowJson
endpoint={supervisorEndpointBase}
transform={x => cleanSpec(x, true)}
diff --git a/web-console/src/dialogs/task-table-action-dialog/task-table-action-dialog.tsx b/web-console/src/dialogs/task-table-action-dialog/task-table-action-dialog.tsx
index a0a5dbbf13f..9edc5d996f4 100644
--- a/web-console/src/dialogs/task-table-action-dialog/task-table-action-dialog.tsx
+++ b/web-console/src/dialogs/task-table-action-dialog/task-table-action-dialog.tsx
@@ -25,18 +25,20 @@ import type { BasicAction } from '../../utils/basic-action';
import type { SideButtonMetaData } from '../table-action-dialog/table-action-dialog';
import { TableActionDialog } from '../table-action-dialog/table-action-dialog';
+type TaskTableActionDialogTab = 'status' | 'report' | 'spec' | 'log';
+
interface TaskTableActionDialogProps {
taskId: string;
actions: BasicAction[];
- onClose: () => void;
status: string;
+ onClose(): void;
}
export const TaskTableActionDialog = React.memo(function TaskTableActionDialog(
props: TaskTableActionDialogProps,
) {
const { taskId, actions, onClose, status } = props;
- const [activeTab, setActiveTab] = useState('status');
+ const [activeTab, setActiveTab] = useState<TaskTableActionDialogTab>('status');
const taskTableSideButtonMetadata: SideButtonMetaData[] = [
{
@@ -45,21 +47,21 @@ export const TaskTableActionDialog = React.memo(function TaskTableActionDialog(
active: activeTab === 'status',
onClick: () => setActiveTab('status'),
},
- {
- icon: 'align-left',
- text: 'Payload',
- active: activeTab === 'payload',
- onClick: () => setActiveTab('payload'),
- },
{
icon: 'comparison',
text: 'Reports',
- active: activeTab === 'reports',
- onClick: () => setActiveTab('reports'),
+ active: activeTab === 'report',
+ onClick: () => setActiveTab('report'),
+ },
+ {
+ icon: 'align-left',
+ text: 'Spec',
+ active: activeTab === 'spec',
+ onClick: () => setActiveTab('spec'),
},
{
icon: 'align-justify',
- text: 'Logs',
+ text: 'Log',
active: activeTab === 'log',
onClick: () => setActiveTab('log'),
},
@@ -80,20 +82,20 @@ export const TaskTableActionDialog = React.memo(function TaskTableActionDialog(
downloadFilename={`task-status-${taskId}.json`}
/>
)}
- {activeTab === 'payload' && (
- <ShowJson
- endpoint={taskEndpointBase}
- transform={x => deepGet(x, 'payload') || x}
- downloadFilename={`task-payload-${taskId}.json`}
- />
- )}
- {activeTab === 'reports' && (
+ {activeTab === 'report' && (
<ShowJson
endpoint={`${taskEndpointBase}/reports`}
transform={x => deepGet(x, 'ingestionStatsAndErrors.payload') || x}
downloadFilename={`task-reports-${taskId}.json`}
/>
)}
+ {activeTab === 'spec' && (
+ <ShowJson
+ endpoint={taskEndpointBase}
+ transform={x => deepGet(x, 'payload') || x}
+ downloadFilename={`task-payload-${taskId}.json`}
+ />
+ )}
{activeTab === 'log' && (
<ShowLog
tail={status === 'RUNNING'}
diff --git a/web-console/src/druid-models/supervisor-status/supervisor-status.ts b/web-console/src/druid-models/supervisor-status/supervisor-status.ts
index bdbc3187736..d76ad19cc14 100644
--- a/web-console/src/druid-models/supervisor-status/supervisor-status.ts
+++ b/web-console/src/druid-models/supervisor-status/supervisor-status.ts
@@ -42,20 +42,27 @@ export interface SupervisorStatus {
healthy: boolean;
state: string;
detailedState: string;
- recentErrors: any[];
+ recentErrors: SupervisorError[];
};
}
export interface SupervisorStatusTask {
id: string;
startingOffsets: SupervisorOffsetMap;
- startTime: '2024-04-12T21:35:34.834Z';
+ startTime: string;
remainingSeconds: number;
type: string;
currentOffsets: SupervisorOffsetMap;
lag: SupervisorOffsetMap;
}
+export interface SupervisorError {
+ timestamp: string;
+ exceptionClass: string;
+ message: string;
+ streamException: boolean;
+}
+
export type SupervisorStats = Record<string, Record<string, RowStats>>;
export type RowStatsKey = 'totals' | '1m' | '5m' | '15m';
diff --git a/web-console/src/views/load-data-view/info-messages.tsx b/web-console/src/views/load-data-view/info-messages.tsx
index b88cf8a70c2..ad9e96667db 100644
--- a/web-console/src/views/load-data-view/info-messages.tsx
+++ b/web-console/src/views/load-data-view/info-messages.tsx
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-import { Button, Callout, Code, FormGroup, Intent } from '@blueprintjs/core';
+import { Button, Callout, Code, FormGroup, Intent, Tag } from '@blueprintjs/core';
import React from 'react';
import { ExternalLink, LearnMore } from '../../components';
@@ -236,8 +236,8 @@ export const AppendToExistingIssue = React.memo(function AppendToExistingIssue(
<FormGroup>
<Callout intent={Intent.DANGER}>
<p>
- Only <Code>dynamic</Code> partitioning supports <Code>appendToExisting: true</Code>. You
- have currently selected <Code>{partitionsSpecType}</Code> partitioning.
+ Only <Tag minimal>dynamic</Tag> partitioning supports <Code>appendToExisting: true</Code>.
+ You have currently selected <Tag minimal>{partitionsSpecType}</Tag> partitioning.
</p>
<Button
intent={Intent.SUCCESS}
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx
index fdb45420e0a..d95943b4f1c 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -34,6 +34,7 @@ import {
Radio,
RadioGroup,
Switch,
+ Tag,
TextArea,
} from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
@@ -3073,8 +3074,9 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
<p>Your partitioning and sorting configuration is uncommon.</p>
<p>
For best performance the first dimension in your schema (
- <Code>{firstDimensionName}</Code>), which is what the data will be primarily sorted
- on, commonly matches the partitioning dimension (<Code>{partitionDimension}</Code>).
+ <Tag minimal>{firstDimensionName}</Tag>), which is what the data will be primarily
+ sorted on, commonly matches the partitioning dimension (
+ <Tag minimal>{partitionDimension}</Tag>).
</p>
<p>
<Button
@@ -3452,11 +3454,11 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
<p>
You have enabled type-aware schema discovery (
<Code>useSchemaDiscovery: true</Code>) to ingest data into the existing
- datasource <Code>{datasource}</Code>.
+ datasource <Tag minimal>{datasource}</Tag>.
</p>
<p>
If you used string-based schema discovery when first ingesting data to{' '}
- <Code>{datasource}</Code>, using type-aware schema discovery now can cause
+ <Tag minimal>{datasource}</Tag>, using type-aware schema discovery now can cause
problems with the values multi-value string dimensions.
</p>
<p>
diff --git a/web-console/src/views/segments-view/segments-view.tsx b/web-console/src/views/segments-view/segments-view.tsx
index 44d59f6fa95..ba4b84b76c4 100644
--- a/web-console/src/views/segments-view/segments-view.tsx
+++ b/web-console/src/views/segments-view/segments-view.tsx
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-import { Button, ButtonGroup, Code, Intent, Label, MenuItem, Switch } from '@blueprintjs/core';
+import { Button, ButtonGroup, Intent, Label, MenuItem, Switch, Tag } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import { C, L, SqlComparison, SqlExpression } from '@druid-toolkit/query';
import classNames from 'classnames';
@@ -919,7 +919,7 @@ END AS "time_span"`,
);
return resp.data;
}}
- confirmButtonText="Drop Segment"
+ confirmButtonText="Drop segment"
successText="Segment drop request acknowledged, next time the coordinator runs segment will be dropped"
failText="Could not drop segment"
intent={Intent.DANGER}
@@ -931,7 +931,7 @@ END AS "time_span"`,
}}
>
<p>
- Are you sure you want to drop segment <Code>{terminateSegmentId}</Code>?
+ Are you sure you want to drop segment <Tag minimal>{terminateSegmentId}</Tag>?
</p>
<p>This action is not reversible.</p>
</AsyncActionDialog>
diff --git a/web-console/src/views/supervisors-view/supervisors-view.scss b/web-console/src/views/supervisors-view/supervisors-view.scss
index edf04bc4e0d..6db2b646d95 100644
--- a/web-console/src/views/supervisors-view/supervisors-view.scss
+++ b/web-console/src/views/supervisors-view/supervisors-view.scss
@@ -28,5 +28,9 @@
top: $view-control-bar-height + $standard-padding;
bottom: 0;
width: 100%;
+
+ .title-button {
+ cursor: pointer;
+ }
}
}
diff --git a/web-console/src/views/supervisors-view/supervisors-view.tsx b/web-console/src/views/supervisors-view/supervisors-view.tsx
index 6e872792220..0483cbd2184 100644
--- a/web-console/src/views/supervisors-view/supervisors-view.tsx
+++ b/web-console/src/views/supervisors-view/supervisors-view.tsx
@@ -16,9 +16,11 @@
* limitations under the License.
*/
-import { Button, Code, Intent, Menu, MenuItem, Position } from '@blueprintjs/core';
+import { Icon, Intent, Menu, MenuItem, Position, Tag } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import { Popover2 } from '@blueprintjs/popover2';
+import * as JSONBig from 'json-bigint-native';
+import type { JSX } from 'react';
import React from 'react';
import type { Filter } from 'react-table';
import ReactTable from 'react-table';
@@ -42,7 +44,12 @@ import {
SupervisorTableActionDialog,
} from '../../dialogs';
import { SupervisorResetOffsetsDialog } from '../../dialogs/supervisor-reset-offsets-dialog/supervisor-reset-offsets-dialog';
-import type { QueryWithContext, RowStatsKey, SupervisorStatus } from '../../druid-models';
+import type {
+ IngestionSpec,
+ QueryWithContext,
+ RowStatsKey,
+ SupervisorStatus,
+} from '../../druid-models';
import { getTotalSupervisorStats } from '../../druid-models';
import type { Capabilities } from '../../helpers';
import {
@@ -64,6 +71,7 @@ import {
hasPopoverOpen,
LocalStorageBackedVisibility,
LocalStorageKeys,
+ nonEmptyArray,
oneOf,
pluralIfNeeded,
queryDruidSql,
@@ -81,16 +89,16 @@ const supervisorTableColumns: string[] = [
'Type',
'Topic/Stream',
'Status',
- 'Active tasks',
+ 'Configured tasks',
+ 'Running tasks',
'Aggregate lag',
'Stats',
ACTION_COLUMN_LABEL,
];
-const ROW_STATS_KEYS: RowStatsKey[] = ['totals', '1m', '5m', '15m'];
+const ROW_STATS_KEYS: RowStatsKey[] = ['1m', '5m', '15m'];
function getRowStatsKeyTitle(key: RowStatsKey) {
- if (key === 'totals') return 'Total';
return `Rate over past ${pluralIfNeeded(parseInt(key, 10), 'minute')}`;
}
@@ -104,6 +112,7 @@ interface SupervisorQueryResultRow {
type: string;
source: string;
detailed_state: string;
+ spec?: IngestionSpec;
suspended: boolean;
status?: SupervisorStatus;
stats?: any;
@@ -182,14 +191,6 @@ export class SupervisorsView extends React.PureComponent<
SupervisorQueryResultRow[]
>;
- static SUPERVISOR_SQL_BASE = `WITH s AS (SELECT
- "supervisor_id",
- "type",
- "source",
- CASE WHEN "suspended" = 0 THEN "detailed_state" ELSE 'SUSPENDED' END AS "detailed_state",
- "suspended" = 1 AS "suspended"
-FROM "sys"."supervisors")`;
-
constructor(props: SupervisorsViewProps) {
super(props);
@@ -219,7 +220,14 @@ FROM "sys"."supervisors")`;
let supervisors: SupervisorQueryResultRow[];
if (capabilities.hasSql()) {
const sqlQuery = assemble(
- SupervisorsView.SUPERVISOR_SQL_BASE,
+ 'WITH s AS (SELECT',
+ ' "supervisor_id",',
+ ' "type",',
+ ' "source",',
+ ` CASE WHEN "suspended" = 0 THEN "detailed_state" ELSE 'SUSPENDED' END AS "detailed_state",`,
+ visibleColumns.shown('Configured tasks') ? ' "spec",' : undefined,
+ ' "suspended" = 1 AS "suspended"',
+ 'FROM "sys"."supervisors")',
'SELECT *',
'FROM s',
filtered.length
@@ -236,6 +244,13 @@ FROM "sys"."supervisors")`;
},
cancelToken,
);
+
+ for (const supervisor of supervisors) {
+ const spec: any = supervisor.spec;
+ if (typeof spec === 'string') {
+ supervisor.spec = JSONBig.parse(spec);
+ }
+ }
} else if (capabilities.hasOverlordAccess()) {
const supervisorList = (
await Api.instance.get('/druid/indexer/v1/supervisor?full', { cancelToken })
@@ -253,6 +268,7 @@ FROM "sys"."supervisors")`;
'n/a',
state: deepGet(sup, 'state'),
detailed_state: deepGet(sup, 'detailedState'),
+ spec: sup.spec,
suspended: Boolean(deepGet(sup, 'suspended')),
};
});
@@ -272,7 +288,7 @@ FROM "sys"."supervisors")`;
}
if (capabilities.hasOverlordAccess()) {
- if (visibleColumns.shown('Active tasks') || visibleColumns.shown('Aggregate lag')) {
+ if (visibleColumns.shown('Running tasks') || visibleColumns.shown('Aggregate lag')) {
try {
for (const supervisor of supervisors) {
cancelToken.throwIfRequested();
@@ -451,7 +467,7 @@ FROM "sys"."supervisors")`;
}}
>
<p>
- Are you sure you want to resume supervisor <Code>{resumeSupervisorId}</Code>?
+ Are you sure you want to resume supervisor <Tag minimal>{resumeSupervisorId}</Tag>?
</p>
</AsyncActionDialog>
);
@@ -482,7 +498,7 @@ FROM "sys"."supervisors")`;
}}
>
<p>
- Are you sure you want to suspend supervisor <Code>{suspendSupervisorId}</Code>?
+ Are you sure you want to suspend supervisor <Tag minimal>{suspendSupervisorId}</Tag>?
</p>
</AsyncActionDialog>
);
@@ -527,17 +543,20 @@ FROM "sys"."supervisors")`;
this.supervisorQueryManager.rerunLastQuery();
}}
warningChecks={[
- `I understand that resetting ${resetSupervisorId} will clear checkpoints and therefore lead to data loss or duplication.`,
+ <>
+ I understand that resetting <Tag minimal>{resetSupervisorId}</Tag> will clear
+ checkpoints and may lead to data loss or duplication.
+ </>,
'I understand that this operation cannot be undone.',
]}
>
<p>
- Are you sure you want to hard reset supervisor <Code>{resetSupervisorId}</Code>?
+ Are you sure you want to hard reset supervisor <Tag minimal>{resetSupervisorId}</Tag>?
</p>
- <p>Hard resetting a supervisor will lead to data loss or data duplication.</p>
+ <p>Hard resetting a supervisor may lead to data loss or data duplication.</p>
<p>
- The reason for using this operation is to recover from a state in which the supervisor
- ceases operating due to missing offsets.
+ Use this operation to restore functionality when the supervisor stops operating due to
+ missing offsets.
</p>
</AsyncActionDialog>
);
@@ -568,7 +587,7 @@ FROM "sys"."supervisors")`;
}}
>
<p>
- Are you sure you want to terminate supervisor <Code>{terminateSupervisorId}</Code>?
+ Are you sure you want to terminate supervisor <Tag minimal>{terminateSupervisorId}</Tag>?
</p>
<p>This action is not reversible.</p>
</AsyncActionDialog>
@@ -673,29 +692,69 @@ FROM "sys"."supervisors")`;
show: visibleColumns.shown('Status'),
},
{
- Header: 'Active tasks',
- id: 'active_tasks',
+ Header: 'Configured tasks',
+ id: 'configured_tasks',
width: 150,
- accessor: 'status.payload.activeTasks',
+ accessor: 'spec',
+ filterable: false,
+ sortable: false,
+ className: 'padded',
+ Cell: ({ value }) => {
+ if (!value) return null;
+ const taskCount = deepGet(value, 'spec.ioConfig.taskCount');
+ const replicas = deepGet(value, 'spec.ioConfig.replicas');
+ if (typeof taskCount !== 'number' || typeof replicas !== 'number') return null;
+ return (
+ <div>
+ <div>{formatInteger(taskCount * replicas)}</div>
+ <div>
+ {replicas === 1
+ ? '(no replication)'
+ : `(${pluralIfNeeded(taskCount, 'task')} × ${pluralIfNeeded(
+ replicas,
+ 'replica',
+ )})`}
+ </div>
+ </div>
+ );
+ },
+ show: visibleColumns.shown('Configured tasks'),
+ },
+ {
+ Header: 'Running tasks',
+ id: 'running_tasks',
+ width: 150,
+ accessor: 'status.payload',
filterable: false,
sortable: false,
Cell: ({ value, original }) => {
if (original.suspended) return;
+ let label: string | JSX.Element;
+ const { activeTasks, publishingTasks } = value || {};
+ if (Array.isArray(activeTasks)) {
+ label = pluralIfNeeded(activeTasks.length, 'active task');
+ if (nonEmptyArray(publishingTasks)) {
+ label = (
+ <>
+ <div>{label}</div>
+ <div>{pluralIfNeeded(publishingTasks.length, 'publishing task')}</div>
+ </>
+ );
+ }
+ } else {
+ label = 'n/a';
+ }
return (
<TableClickableCell
onClick={() => goToTasks(original.supervisor_id, `index_${original.type}`)}
hoverIcon={IconNames.ARROW_TOP_RIGHT}
title="Go to tasks"
>
- {typeof value === 'undefined'
- ? 'n/a'
- : value.length > 0
- ? pluralIfNeeded(value.length, 'running task')
- : `No running tasks`}
+ {label}
</TableClickableCell>
);
},
- show: visibleColumns.shown('Active tasks'),
+ show: visibleColumns.shown('Running tasks'),
},
{
Header: 'Aggregate lag',
@@ -708,7 +767,30 @@ FROM "sys"."supervisors")`;
Cell: ({ value }) => formatInteger(value),
},
{
- Header: twoLines('Stats', <i>{getRowStatsKeyTitle(statsKey)}</i>),
+ Header: twoLines(
+ 'Stats',
+ <Popover2
+ position={Position.BOTTOM}
+ content={
+ <Menu>
+ {ROW_STATS_KEYS.map(k => (
+ <MenuItem
+ key={k}
+ icon={checkedCircleIcon(k === statsKey)}
+ text={getRowStatsKeyTitle(k)}
+ onClick={() => {
+ this.setState({ statsKey: k });
+ }}
+ />
+ ))}
+ </Menu>
+ }
+ >
+ <i className="title-button">
+ {getRowStatsKeyTitle(statsKey)} <Icon icon={IconNames.CARET_DOWN} />
+ </i>
+ </Popover2>,
+ ),
id: 'stats',
width: 220,
filterable: false,
@@ -889,7 +971,6 @@ FROM "sys"."supervisors")`;
supervisorTableActionDialogId,
supervisorTableActionDialogActions,
visibleColumns,
- statsKey,
} = this.state;
return (
@@ -902,28 +983,6 @@ FROM "sys"."supervisors")`;
this.supervisorQueryManager.rerunLastQuery(auto);
}}
/>
- <Popover2
- position={Position.BOTTOM}
- content={
- <Menu>
- {ROW_STATS_KEYS.map(k => (
- <MenuItem
- key={k}
- icon={checkedCircleIcon(k === statsKey)}
- text={getRowStatsKeyTitle(k)}
- onClick={() => {
- this.setState({ statsKey: k });
- }}
- />
- ))}
- </Menu>
- }
- >
- <Button
- text={`Stats: ${getRowStatsKeyTitle(statsKey)}`}
- rightIcon={IconNames.CARET_DOWN}
- />
- </Popover2>
{this.renderBulkSupervisorActions()}
<TableColumnSelector
columns={supervisorTableColumns}
diff --git a/web-console/src/views/tasks-view/tasks-view.tsx b/web-console/src/views/tasks-view/tasks-view.tsx
index 795c9908412..2be3575b9e1 100644
--- a/web-console/src/views/tasks-view/tasks-view.tsx
+++ b/web-console/src/views/tasks-view/tasks-view.tsx
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-import { Button, ButtonGroup, Intent, Label, MenuItem } from '@blueprintjs/core';
+import { Button, ButtonGroup, Intent, Label, MenuItem, Tag } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import React from 'react';
import type { Filter } from 'react-table';
@@ -310,7 +310,9 @@ ORDER BY
this.taskQueryManager.rerunLastQuery();
}}
>
- <p>{`Are you sure you want to kill task '${killTaskId}'?`}</p>
+ <p>
+ Are you sure you want to kill task <Tag minimal>{killTaskId}</Tag>?
+ </p>
</AsyncActionDialog>
);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org
(druid) 01/04: add rate and stats
Posted by vo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch better_supervisor_view
in repository https://gitbox.apache.org/repos/asf/druid.git
commit 151a2060981328881e90e31103848ec45e87abbd
Author: Vadim Ogievetsky <va...@ogievetsky.com>
AuthorDate: Thu Apr 18 12:32:16 2024 -0700
add rate and stats
---
.../src/components/header-bar/header-bar.scss | 5 +
.../src/components/timed-button/timed-button.tsx | 4 +-
.../supervisor-statistics-table.spec.tsx | 4 +
.../supervisor-statistics-table.tsx | 56 ++--
.../supervisor-status/supervisor-status.ts | 54 ++++
web-console/src/utils/druid-query.ts | 9 +-
web-console/src/utils/general.tsx | 12 +
web-console/src/utils/table-helpers.ts | 19 ++
.../src/views/segments-view/segments-view.tsx | 20 +-
.../views/supervisors-view/supervisors-view.tsx | 312 +++++++++++++++------
10 files changed, 350 insertions(+), 145 deletions(-)
diff --git a/web-console/src/components/header-bar/header-bar.scss b/web-console/src/components/header-bar/header-bar.scss
index 062768a22c4..752cc9bf316 100644
--- a/web-console/src/components/header-bar/header-bar.scss
+++ b/web-console/src/components/header-bar/header-bar.scss
@@ -89,4 +89,9 @@
}
}
}
+
+ .#{$bp-ns}-navbar-group.#{$bp-ns}-align-right {
+ position: absolute;
+ right: 15px;
+ }
}
diff --git a/web-console/src/components/timed-button/timed-button.tsx b/web-console/src/components/timed-button/timed-button.tsx
index 0b339a8d0e5..cb275370b27 100644
--- a/web-console/src/components/timed-button/timed-button.tsx
+++ b/web-console/src/components/timed-button/timed-button.tsx
@@ -25,7 +25,7 @@ import React, { useState } from 'react';
import { useInterval } from '../../hooks';
import type { LocalStorageKeys } from '../../utils';
-import { isInBackground, localStorageGet, localStorageSet } from '../../utils';
+import { checkedCircleIcon, isInBackground, localStorageGet, localStorageSet } from '../../utils';
export interface DelayLabel {
label: string;
@@ -84,7 +84,7 @@ export const TimedButton = React.memo(function TimedButton(props: TimedButtonPro
{delays.map(({ label, delay }, i) => (
<MenuItem
key={i}
- icon={selectedDelay === delay ? IconNames.SELECTION : IconNames.CIRCLE}
+ icon={checkedCircleIcon(selectedDelay === delay)}
text={label}
onClick={() => handleSelection(delay)}
/>
diff --git a/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/supervisor-statistics-table.spec.tsx b/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/supervisor-statistics-table.spec.tsx
index 3ea907dc0cf..8934f761207 100644
--- a/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/supervisor-statistics-table.spec.tsx
+++ b/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/supervisor-statistics-table.spec.tsx
@@ -72,18 +72,21 @@ describe('SupervisorStatisticsTable', () => {
buildSegments: {
'5m': {
processed: 3.5455993615040584,
+ processedBytes: 10,
unparseable: 0,
thrownAway: 0,
processedWithError: 0,
},
'15m': {
processed: 5.544749689510444,
+ processedBytes: 20,
unparseable: 0,
thrownAway: 0,
processedWithError: 0,
},
'1m': {
processed: 4.593670088770785,
+ processedBytes: 30,
unparseable: 0,
thrownAway: 0,
processedWithError: 0,
@@ -93,6 +96,7 @@ describe('SupervisorStatisticsTable', () => {
totals: {
buildSegments: {
processed: 7516,
+ processedBytes: 60,
processedWithError: 0,
thrownAway: 0,
unparseable: 0,
diff --git a/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/supervisor-statistics-table.tsx b/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/supervisor-statistics-table.tsx
index 09749f85253..26cfd62f333 100644
--- a/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/supervisor-statistics-table.tsx
+++ b/web-console/src/dialogs/supervisor-table-action-dialog/supervisor-statistics-table/supervisor-statistics-table.tsx
@@ -22,33 +22,21 @@ import type { CellInfo, Column } from 'react-table';
import ReactTable from 'react-table';
import { Loader } from '../../../components/loader/loader';
+import type { RowStats, RowStatsCounter, SupervisorStats } from '../../../druid-models';
import { useQueryManager } from '../../../hooks';
import { SMALL_TABLE_PAGE_SIZE, SMALL_TABLE_PAGE_SIZE_OPTIONS } from '../../../react-table';
import { Api, UrlBaser } from '../../../singletons';
-import { deepGet } from '../../../utils';
+import { deepGet, formatByteRate, formatBytes, formatInteger, formatRate } from '../../../utils';
import './supervisor-statistics-table.scss';
-export interface TaskSummary {
- totals: Record<string, StatsEntry>;
- movingAverages: Record<string, Record<string, StatsEntry>>;
-}
-
-export interface StatsEntry {
- processed?: number;
- processedWithError?: number;
- thrownAway?: number;
- unparseable?: number;
- [key: string]: number | undefined;
-}
-
export interface SupervisorStatisticsTableRow {
taskId: string;
- summary: TaskSummary;
+ summary: RowStats;
}
export function normalizeSupervisorStatisticsResults(
- data: Record<string, Record<string, TaskSummary>>,
+ data: SupervisorStats,
): SupervisorStatisticsTableRow[] {
return Object.values(data).flatMap(v => Object.keys(v).map(k => ({ taskId: k, summary: v[k] })));
}
@@ -66,21 +54,29 @@ export const SupervisorStatisticsTable = React.memo(function SupervisorStatistic
const [supervisorStatisticsState] = useQueryManager<null, SupervisorStatisticsTableRow[]>({
processQuery: async () => {
- const resp = await Api.instance.get(endpoint);
+ const resp = await Api.instance.get<SupervisorStats>(endpoint);
return normalizeSupervisorStatisticsResults(resp.data);
},
initQuery: null,
});
- function renderCell(cell: CellInfo) {
- const cellValue = cell.value;
- if (!cellValue) {
- return <div>No data found</div>;
- }
+ function renderCounters(cell: CellInfo, isRate: boolean) {
+ const c: RowStatsCounter = cell.value;
+ if (!c) return null;
- return Object.keys(cellValue)
- .sort()
- .map(key => <div key={key}>{`${key}: ${Number(cellValue[key]).toFixed(1)}`}</div>);
+ const formatNumber = isRate ? formatRate : formatInteger;
+ const formatData = isRate ? formatByteRate : formatBytes;
+ const bytes = c.processedBytes ? ` (${formatData(c.processedBytes)})` : '';
+ return (
+ <div>
+ <div>{`Processed: ${formatNumber(c.processed)}${bytes}`}</div>
+ {Boolean(c.processedWithError) && (
+ <div>Processed with error: {formatNumber(c.processedWithError)}</div>
+ )}
+ {Boolean(c.thrownAway) && <div>Thrown away: {formatNumber(c.thrownAway)}</div>}
+ {Boolean(c.unparseable) && <div>Unparseable: {formatNumber(c.unparseable)}</div>}
+ </div>
+ );
}
function renderTable() {
@@ -98,9 +94,9 @@ export const SupervisorStatisticsTable = React.memo(function SupervisorStatistic
className: 'padded',
width: 200,
accessor: d => {
- return deepGet(d, 'summary.totals.buildSegments') as StatsEntry;
+ return deepGet(d, 'summary.totals.buildSegments') as RowStatsCounter;
},
- Cell: renderCell,
+ Cell: c => renderCounters(c, false),
},
];
@@ -118,10 +114,8 @@ export const SupervisorStatisticsTable = React.memo(function SupervisorStatistic
id: interval,
className: 'padded',
width: 200,
- accessor: d => {
- return deepGet(d, `summary.movingAverages.buildSegments.${interval}`);
- },
- Cell: renderCell,
+ accessor: `summary.movingAverages.buildSegments.${interval}`,
+ Cell: c => renderCounters(c, true),
};
}),
);
diff --git a/web-console/src/druid-models/supervisor-status/supervisor-status.ts b/web-console/src/druid-models/supervisor-status/supervisor-status.ts
index b25c9f5fb1f..bdbc3187736 100644
--- a/web-console/src/druid-models/supervisor-status/supervisor-status.ts
+++ b/web-console/src/druid-models/supervisor-status/supervisor-status.ts
@@ -16,7 +16,10 @@
* limitations under the License.
*/
+import { sum } from 'd3-array';
+
import type { NumberLike } from '../../utils';
+import { compact, deepGet } from '../../utils';
export type SupervisorOffsetMap = Record<string, NumberLike>;
@@ -52,3 +55,54 @@ export interface SupervisorStatusTask {
currentOffsets: SupervisorOffsetMap;
lag: SupervisorOffsetMap;
}
+
+export type SupervisorStats = Record<string, Record<string, RowStats>>;
+
+export type RowStatsKey = 'totals' | '1m' | '5m' | '15m';
+
+export interface RowStats {
+ movingAverages: {
+ buildSegments: {
+ '1m': RowStatsCounter;
+ '5m': RowStatsCounter;
+ '15m': RowStatsCounter;
+ };
+ };
+ totals: {
+ buildSegments: RowStatsCounter;
+ };
+}
+
+export interface RowStatsCounter {
+ processed: number;
+ processedBytes: number;
+ processedWithError: number;
+ thrownAway: number;
+ unparseable: number;
+}
+
+function aggregateRowStatsCounter(rowStats: RowStatsCounter[]): RowStatsCounter {
+ return {
+ processed: sum(rowStats, d => d.processed),
+ processedBytes: sum(rowStats, d => d.processedBytes),
+ processedWithError: sum(rowStats, d => d.processedWithError),
+ thrownAway: sum(rowStats, d => d.thrownAway),
+ unparseable: sum(rowStats, d => d.unparseable),
+ };
+}
+
+function getRowStatsCounter(rowStats: RowStats, key: RowStatsKey): RowStatsCounter | undefined {
+ if (key === 'totals') {
+ return deepGet(rowStats, 'totals.buildSegments');
+ } else {
+ return deepGet(rowStats, `movingAverages.buildSegments.${key}`);
+ }
+}
+
+export function getTotalSupervisorStats(stats: SupervisorStats, key: RowStatsKey): RowStatsCounter {
+ return aggregateRowStatsCounter(
+ compact(
+ Object.values(stats).flatMap(s => Object.values(s).map(rs => getRowStatsCounter(rs, key))),
+ ),
+ );
+}
diff --git a/web-console/src/utils/druid-query.ts b/web-console/src/utils/druid-query.ts
index c94bfca3d1c..15410329704 100644
--- a/web-console/src/utils/druid-query.ts
+++ b/web-console/src/utils/druid-query.ts
@@ -17,7 +17,7 @@
*/
import { C } from '@druid-toolkit/query';
-import type { AxiosResponse } from 'axios';
+import type { AxiosResponse, CancelToken } from 'axios';
import axios from 'axios';
import { Api } from '../singletons';
@@ -329,10 +329,13 @@ export async function queryDruidRune(runeQuery: Record<string, any>): Promise<an
return runeResultResp.data;
}
-export async function queryDruidSql<T = any>(sqlQueryPayload: Record<string, any>): Promise<T[]> {
+export async function queryDruidSql<T = any>(
+ sqlQueryPayload: Record<string, any>,
+ cancelToken?: CancelToken,
+): Promise<T[]> {
let sqlResultResp: AxiosResponse;
try {
- sqlResultResp = await Api.instance.post('/druid/v2/sql', sqlQueryPayload);
+ sqlResultResp = await Api.instance.post('/druid/v2/sql', sqlQueryPayload, { cancelToken });
} catch (e) {
throw new Error(getDruidErrorMessage(e));
}
diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx
index 3a770c67630..b4537a63e08 100644
--- a/web-console/src/utils/general.tsx
+++ b/web-console/src/utils/general.tsx
@@ -239,14 +239,26 @@ export function formatNumber(n: NumberLike): string {
return n.toLocaleString('en-US', { maximumFractionDigits: 20 });
}
+export function formatRate(n: NumberLike) {
+ return numeral(n).format('0,0.0') + '/s';
+}
+
export function formatBytes(n: NumberLike): string {
return numeral(n).format('0.00 b');
}
+export function formatByteRate(n: NumberLike): string {
+ return numeral(n).format('0.00 b') + '/s';
+}
+
export function formatBytesCompact(n: NumberLike): string {
return numeral(n).format('0.00b');
}
+export function formatByteRateCompact(n: NumberLike): string {
+ return numeral(n).format('0.00b') + '/s';
+}
+
export function formatMegabytes(n: NumberLike): string {
return numeral(Number(n) / 1048576).format('0,0.0');
}
diff --git a/web-console/src/utils/table-helpers.ts b/web-console/src/utils/table-helpers.ts
index e864aef131f..7eedd1acaab 100644
--- a/web-console/src/utils/table-helpers.ts
+++ b/web-console/src/utils/table-helpers.ts
@@ -17,6 +17,8 @@
*/
import type { QueryResult } from '@druid-toolkit/query';
+import { C } from '@druid-toolkit/query';
+import type { Filter } from 'react-table';
import { filterMap, formatNumber, oneOf } from './general';
import { deepSet } from './object-change';
@@ -56,3 +58,20 @@ export function getNumericColumnBraces(
return numericColumnBraces;
}
+
+export interface Sorted {
+ id: string;
+ desc: boolean;
+}
+
+export interface TableState {
+ page: number;
+ pageSize: number;
+ filtered: Filter[];
+ sorted: Sorted[];
+}
+
+export function sortedToOrderByClause(sorted: Sorted[]): string | undefined {
+ if (!sorted.length) return;
+ return 'ORDER BY ' + sorted.map(sort => `${C(sort.id)} ${sort.desc ? 'DESC' : 'ASC'}`).join(', ');
+}
diff --git a/web-console/src/views/segments-view/segments-view.tsx b/web-console/src/views/segments-view/segments-view.tsx
index ae40a8d641e..44d59f6fa95 100644
--- a/web-console/src/views/segments-view/segments-view.tsx
+++ b/web-console/src/views/segments-view/segments-view.tsx
@@ -53,7 +53,7 @@ import {
STANDARD_TABLE_PAGE_SIZE_OPTIONS,
} from '../../react-table';
import { Api } from '../../singletons';
-import type { NumberLike } from '../../utils';
+import type { NumberLike, TableState } from '../../utils';
import {
compact,
countBy,
@@ -69,6 +69,7 @@ import {
queryDruidSql,
QueryManager,
QueryState,
+ sortedToOrderByClause,
twoLines,
} from '../../utils';
import type { BasicAction } from '../../utils/basic-action';
@@ -134,23 +135,6 @@ function formatRangeDimensionValue(dimension: any, value: any): string {
return `${C(String(dimension))}=${L(String(value))}`;
}
-interface Sorted {
- id: string;
- desc: boolean;
-}
-
-function sortedToOrderByClause(sorted: Sorted[]): string | undefined {
- if (!sorted.length) return;
- return 'ORDER BY ' + sorted.map(sort => `${C(sort.id)} ${sort.desc ? 'DESC' : 'ASC'}`).join(', ');
-}
-
-interface TableState {
- page: number;
- pageSize: number;
- filtered: Filter[];
- sorted: Sorted[];
-}
-
interface SegmentsQuery extends TableState {
visibleColumns: LocalStorageBackedVisibility;
capabilities: Capabilities;
diff --git a/web-console/src/views/supervisors-view/supervisors-view.tsx b/web-console/src/views/supervisors-view/supervisors-view.tsx
index fef74fbeebe..6e872792220 100644
--- a/web-console/src/views/supervisors-view/supervisors-view.tsx
+++ b/web-console/src/views/supervisors-view/supervisors-view.tsx
@@ -16,8 +16,9 @@
* limitations under the License.
*/
-import { Code, Intent, MenuItem } from '@blueprintjs/core';
+import { Button, Code, Intent, Menu, MenuItem, Position } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
+import { Popover2 } from '@blueprintjs/popover2';
import React from 'react';
import type { Filter } from 'react-table';
import ReactTable from 'react-table';
@@ -41,23 +42,34 @@ import {
SupervisorTableActionDialog,
} from '../../dialogs';
import { SupervisorResetOffsetsDialog } from '../../dialogs/supervisor-reset-offsets-dialog/supervisor-reset-offsets-dialog';
-import type { QueryWithContext } from '../../druid-models';
+import type { QueryWithContext, RowStatsKey, SupervisorStatus } from '../../druid-models';
+import { getTotalSupervisorStats } from '../../druid-models';
import type { Capabilities } from '../../helpers';
-import { SMALL_TABLE_PAGE_SIZE, SMALL_TABLE_PAGE_SIZE_OPTIONS } from '../../react-table';
+import {
+ SMALL_TABLE_PAGE_SIZE,
+ SMALL_TABLE_PAGE_SIZE_OPTIONS,
+ sqlQueryCustomTableFilter,
+} from '../../react-table';
import { Api, AppToaster } from '../../singletons';
+import type { TableState } from '../../utils';
import {
+ assemble,
+ checkedCircleIcon,
deepGet,
+ formatByteRate,
+ formatBytes,
+ formatInteger,
+ formatRate,
getDruidErrorMessage,
- groupByAsMap,
hasPopoverOpen,
LocalStorageBackedVisibility,
LocalStorageKeys,
- lookupBy,
oneOf,
pluralIfNeeded,
queryDruidSql,
QueryManager,
QueryState,
+ sortedToOrderByClause,
twoLines,
} from '../../utils';
import type { BasicAction } from '../../utils/basic-action';
@@ -69,11 +81,20 @@ const supervisorTableColumns: string[] = [
'Type',
'Topic/Stream',
'Status',
- 'Running tasks',
+ 'Active tasks',
+ 'Aggregate lag',
+ 'Stats',
ACTION_COLUMN_LABEL,
];
-interface SupervisorQuery {
+const ROW_STATS_KEYS: RowStatsKey[] = ['totals', '1m', '5m', '15m'];
+
+function getRowStatsKeyTitle(key: RowStatsKey) {
+ if (key === 'totals') return 'Total';
+ return `Rate over past ${pluralIfNeeded(parseInt(key, 10), 'minute')}`;
+}
+
+interface SupervisorQuery extends TableState {
capabilities: Capabilities;
visibleColumns: LocalStorageBackedVisibility;
}
@@ -84,13 +105,8 @@ interface SupervisorQueryResultRow {
source: string;
detailed_state: string;
suspended: boolean;
- running_tasks?: number;
-}
-
-interface RunningTaskRow {
- datasource: string;
- type: string;
- num_running_tasks: number;
+ status?: SupervisorStatus;
+ stats?: any;
}
export interface SupervisorsViewProps {
@@ -106,6 +122,7 @@ export interface SupervisorsViewProps {
export interface SupervisorsViewState {
supervisorsState: QueryState<SupervisorQueryResultRow[]>;
+ statsKey: RowStatsKey;
resumeSupervisorId?: string;
suspendSupervisorId?: string;
@@ -165,25 +182,20 @@ export class SupervisorsView extends React.PureComponent<
SupervisorQueryResultRow[]
>;
- static SUPERVISOR_SQL = `SELECT
+ static SUPERVISOR_SQL_BASE = `WITH s AS (SELECT
"supervisor_id",
"type",
"source",
CASE WHEN "suspended" = 0 THEN "detailed_state" ELSE 'SUSPENDED' END AS "detailed_state",
"suspended" = 1 AS "suspended"
-FROM "sys"."supervisors"
-ORDER BY "supervisor_id"`;
-
- static RUNNING_TASK_SQL = `SELECT
- "datasource", "type", COUNT(*) AS "num_running_tasks"
-FROM "sys"."tasks" WHERE "status" = 'RUNNING' AND "runner_status" = 'RUNNING'
-GROUP BY 1, 2`;
+FROM "sys"."supervisors")`;
constructor(props: SupervisorsViewProps) {
super(props);
this.state = {
supervisorsState: QueryState.INIT,
+ statsKey: '5m',
showResumeAllSupervisors: false,
showSuspendAllSupervisors: false,
@@ -199,14 +211,35 @@ GROUP BY 1, 2`;
};
this.supervisorQueryManager = new QueryManager({
- processQuery: async ({ capabilities, visibleColumns }) => {
+ processQuery: async (
+ { capabilities, visibleColumns, filtered, sorted, page, pageSize },
+ cancelToken,
+ setIntermediateQuery,
+ ) => {
let supervisors: SupervisorQueryResultRow[];
if (capabilities.hasSql()) {
- supervisors = await queryDruidSql<SupervisorQueryResultRow>({
- query: SupervisorsView.SUPERVISOR_SQL,
- });
+ const sqlQuery = assemble(
+ SupervisorsView.SUPERVISOR_SQL_BASE,
+ 'SELECT *',
+ 'FROM s',
+ filtered.length
+ ? `WHERE ${filtered.map(sqlQueryCustomTableFilter).join(' AND ')}`
+ : undefined,
+ sortedToOrderByClause(sorted),
+ `LIMIT ${pageSize}`,
+ page ? `OFFSET ${page * pageSize}` : undefined,
+ ).join('\n');
+ setIntermediateQuery(sqlQuery);
+ supervisors = await queryDruidSql<SupervisorQueryResultRow>(
+ {
+ query: sqlQuery,
+ },
+ cancelToken,
+ );
} else if (capabilities.hasOverlordAccess()) {
- const supervisorList = (await Api.instance.get('/druid/indexer/v1/supervisor?full')).data;
+ const supervisorList = (
+ await Api.instance.get('/druid/indexer/v1/supervisor?full', { cancelToken })
+ ).data;
if (!Array.isArray(supervisorList)) {
throw new Error(`Unexpected result from /druid/indexer/v1/supervisor?full`);
}
@@ -223,45 +256,64 @@ GROUP BY 1, 2`;
suspended: Boolean(deepGet(sup, 'suspended')),
};
});
+
+ const firstSorted = sorted[0];
+ if (firstSorted) {
+ const { id, desc } = firstSorted;
+ supervisors.sort((s1: any, s2: any) => {
+ return (
+ String(s1[id]).localeCompare(String(s2[id]), undefined, { numeric: true }) *
+ (desc ? -1 : 1)
+ );
+ });
+ }
} else {
throw new Error(`must have SQL or overlord access`);
}
- if (visibleColumns.shown('Running tasks')) {
- try {
- let runningTaskLookup: Record<string, number>;
- if (capabilities.hasSql()) {
- const runningTasks = await queryDruidSql<RunningTaskRow>({
- query: SupervisorsView.RUNNING_TASK_SQL,
+ if (capabilities.hasOverlordAccess()) {
+ if (visibleColumns.shown('Active tasks') || visibleColumns.shown('Aggregate lag')) {
+ try {
+ for (const supervisor of supervisors) {
+ cancelToken.throwIfRequested();
+ supervisor.status = (
+ await Api.instance.get(
+ `/druid/indexer/v1/supervisor/${Api.encodePath(
+ supervisor.supervisor_id,
+ )}/status`,
+ { cancelToken },
+ )
+ ).data;
+ }
+ } catch (e) {
+ AppToaster.show({
+ icon: IconNames.ERROR,
+ intent: Intent.DANGER,
+ message: 'Could not get status',
});
-
- runningTaskLookup = lookupBy(
- runningTasks,
- ({ datasource, type }) => `${datasource}_${type}`,
- ({ num_running_tasks }) => num_running_tasks,
- );
- } else if (capabilities.hasOverlordAccess()) {
- const taskList = (await Api.instance.get(`/druid/indexer/v1/tasks?state=running`))
- .data;
- runningTaskLookup = groupByAsMap(
- taskList,
- (t: any) => `${t.dataSource}_${t.type}`,
- xs => xs.length,
- );
- } else {
- throw new Error(`must have SQL or overlord access`);
}
+ }
- supervisors.forEach(supervisor => {
- supervisor.running_tasks =
- runningTaskLookup[`${supervisor.supervisor_id}_index_${supervisor.type}`] || 0;
- });
- } catch (e) {
- AppToaster.show({
- icon: IconNames.ERROR,
- intent: Intent.DANGER,
- message: 'Could not get running task counts',
- });
+ if (visibleColumns.shown('Stats')) {
+ try {
+ for (const supervisor of supervisors) {
+ cancelToken.throwIfRequested();
+ supervisor.stats = (
+ await Api.instance.get(
+ `/druid/indexer/v1/supervisor/${Api.encodePath(
+ supervisor.supervisor_id,
+ )}/stats`,
+ { cancelToken },
+ )
+ ).data;
+ }
+ } catch (e) {
+ AppToaster.show({
+ icon: IconNames.ERROR,
+ intent: Intent.DANGER,
+ message: 'Could not get stats',
+ });
+ }
}
}
@@ -275,17 +327,27 @@ GROUP BY 1, 2`;
});
}
- componentDidMount(): void {
- const { capabilities } = this.props;
- const { visibleColumns } = this.state;
-
- this.supervisorQueryManager.runQuery({ capabilities, visibleColumns: visibleColumns });
- }
+ private lastTableState: TableState | undefined;
componentWillUnmount(): void {
this.supervisorQueryManager.terminate();
}
+ private readonly fetchData = (tableState?: TableState) => {
+ const { capabilities } = this.props;
+ const { visibleColumns } = this.state;
+ if (tableState) this.lastTableState = tableState;
+ const { page, pageSize, filtered, sorted } = this.lastTableState!;
+ this.supervisorQueryManager.runQuery({
+ page,
+ pageSize,
+ filtered,
+ sorted,
+ visibleColumns,
+ capabilities,
+ });
+ };
+
private readonly closeSpecDialogs = () => {
this.setState({
supervisorSpecDialogOpen: false,
@@ -541,7 +603,7 @@ GROUP BY 1, 2`;
private renderSupervisorTable() {
const { goToTasks, filters, onFiltersChange } = this.props;
- const { supervisorsState, visibleColumns } = this.state;
+ const { supervisorsState, statsKey, visibleColumns } = this.state;
const supervisors = supervisorsState.data || [];
return (
@@ -554,6 +616,9 @@ GROUP BY 1, 2`;
filtered={filters}
onFilteredChange={onFiltersChange}
filterable
+ onFetchData={tableState => {
+ this.fetchData(tableState);
+ }}
defaultPageSize={SMALL_TABLE_PAGE_SIZE}
pageSizeOptions={SMALL_TABLE_PAGE_SIZE_OPTIONS}
showPagination={supervisors.length > SMALL_TABLE_PAGE_SIZE}
@@ -562,7 +627,7 @@ GROUP BY 1, 2`;
Header: twoLines('Supervisor ID', <i>(datasource)</i>),
id: 'supervisor_id',
accessor: 'supervisor_id',
- width: 300,
+ width: 280,
show: visibleColumns.shown('Supervisor ID'),
Cell: ({ value, original }) => (
<TableClickableCell
@@ -576,21 +641,21 @@ GROUP BY 1, 2`;
{
Header: 'Type',
accessor: 'type',
- width: 100,
+ width: 80,
Cell: this.renderSupervisorFilterableCell('type'),
show: visibleColumns.shown('Type'),
},
{
Header: 'Topic/Stream',
accessor: 'source',
- width: 300,
+ width: 200,
Cell: this.renderSupervisorFilterableCell('source'),
show: visibleColumns.shown('Topic/Stream'),
},
{
Header: 'Status',
- id: 'status',
- width: 250,
+ id: 'detailed_state',
+ width: 150,
accessor: 'detailed_state',
Cell: ({ value }) => (
<TableFilterableCell
@@ -608,27 +673,67 @@ GROUP BY 1, 2`;
show: visibleColumns.shown('Status'),
},
{
- Header: 'Running tasks',
- id: 'running_tasks',
+ Header: 'Active tasks',
+ id: 'active_tasks',
width: 150,
- accessor: 'running_tasks',
+ accessor: 'status.payload.activeTasks',
filterable: false,
- Cell: ({ value, original }) => (
- <TableClickableCell
- onClick={() => goToTasks(original.supervisor_id, `index_${original.type}`)}
- hoverIcon={IconNames.ARROW_TOP_RIGHT}
- title="Go to tasks"
- >
- {typeof value === 'undefined'
- ? 'n/a'
- : value > 0
- ? pluralIfNeeded(value, 'running task')
- : original.suspended
- ? ''
- : `No running tasks`}
- </TableClickableCell>
- ),
- show: visibleColumns.shown('Running tasks'),
+ sortable: false,
+ Cell: ({ value, original }) => {
+ if (original.suspended) return;
+ return (
+ <TableClickableCell
+ onClick={() => goToTasks(original.supervisor_id, `index_${original.type}`)}
+ hoverIcon={IconNames.ARROW_TOP_RIGHT}
+ title="Go to tasks"
+ >
+ {typeof value === 'undefined'
+ ? 'n/a'
+ : value.length > 0
+ ? pluralIfNeeded(value.length, 'running task')
+ : `No running tasks`}
+ </TableClickableCell>
+ );
+ },
+ show: visibleColumns.shown('Active tasks'),
+ },
+ {
+ Header: 'Aggregate lag',
+ accessor: 'status.aggregateLag',
+ width: 200,
+ filterable: false,
+ sortable: false,
+ className: 'padded',
+ show: visibleColumns.shown('Aggregate lag'),
+ Cell: ({ value }) => formatInteger(value),
+ },
+ {
+ Header: twoLines('Stats', <i>{getRowStatsKeyTitle(statsKey)}</i>),
+ id: 'stats',
+ width: 220,
+ filterable: false,
+ sortable: false,
+ className: 'padded',
+ accessor: 'stats',
+ Cell: ({ value }) => {
+ if (!value) return;
+ const c = getTotalSupervisorStats(value, statsKey);
+ const isRate = statsKey !== 'totals';
+ const formatNumber = isRate ? formatRate : formatInteger;
+ const formatData = isRate ? formatByteRate : formatBytes;
+ const bytes = c.processedBytes ? ` (${formatData(c.processedBytes)})` : '';
+ return (
+ <div>
+ <div>{`Processed: ${formatNumber(c.processed)}${bytes}`}</div>
+ {Boolean(c.processedWithError) && (
+ <div>Processed with error: {formatNumber(c.processedWithError)}</div>
+ )}
+ {Boolean(c.thrownAway) && <div>Thrown away: {formatNumber(c.thrownAway)}</div>}
+ {Boolean(c.unparseable) && <div>Unparseable: {formatNumber(c.unparseable)}</div>}
+ </div>
+ );
+ },
+ show: visibleColumns.shown('Stats'),
},
{
Header: ACTION_COLUMN_LABEL,
@@ -636,6 +741,7 @@ GROUP BY 1, 2`;
accessor: 'supervisor_id',
width: ACTION_COLUMN_WIDTH,
filterable: false,
+ sortable: false,
Cell: row => {
const id = row.value;
const type = row.original.type;
@@ -657,6 +763,7 @@ GROUP BY 1, 2`;
renderBulkSupervisorActions() {
const { capabilities, goToQuery } = this.props;
+ const lastSupervisorQuery = this.supervisorQueryManager.getLastIntermediateQuery();
return (
<>
@@ -665,7 +772,7 @@ GROUP BY 1, 2`;
<MenuItem
icon={IconNames.APPLICATION}
text="View SQL query for table"
- onClick={() => goToQuery({ queryString: SupervisorsView.SUPERVISOR_SQL })}
+ onClick={() => goToQuery({ queryString: lastSupervisorQuery })}
/>
)}
<MenuItem
@@ -782,6 +889,7 @@ GROUP BY 1, 2`;
supervisorTableActionDialogId,
supervisorTableActionDialogActions,
visibleColumns,
+ statsKey,
} = this.state;
return (
@@ -794,6 +902,28 @@ GROUP BY 1, 2`;
this.supervisorQueryManager.rerunLastQuery(auto);
}}
/>
+ <Popover2
+ position={Position.BOTTOM}
+ content={
+ <Menu>
+ {ROW_STATS_KEYS.map(k => (
+ <MenuItem
+ key={k}
+ icon={checkedCircleIcon(k === statsKey)}
+ text={getRowStatsKeyTitle(k)}
+ onClick={() => {
+ this.setState({ statsKey: k });
+ }}
+ />
+ ))}
+ </Menu>
+ }
+ >
+ <Button
+ text={`Stats: ${getRowStatsKeyTitle(statsKey)}`}
+ rightIcon={IconNames.CARET_DOWN}
+ />
+ </Popover2>
{this.renderBulkSupervisorActions()}
<TableColumnSelector
columns={supervisorTableColumns}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org
(druid) 04/04: add recent errors
Posted by vo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch better_supervisor_view
in repository https://gitbox.apache.org/repos/asf/druid.git
commit 06ef03402b6636fd06389be89f25c24dab2fb2aa
Author: Vadim Ogievetsky <va...@ogievetsky.com>
AuthorDate: Mon Apr 22 15:24:37 2024 -0700
add recent errors
---
.../src/utils/local-storage-backed-visibility.tsx | 4 +-
.../views/datasources-view/datasources-view.tsx | 5 +-
.../src/views/segments-view/segments-view.tsx | 2 +-
.../views/supervisors-view/supervisors-view.scss | 4 ++
.../views/supervisors-view/supervisors-view.tsx | 68 ++++++++++++++++++----
5 files changed, 66 insertions(+), 17 deletions(-)
diff --git a/web-console/src/utils/local-storage-backed-visibility.tsx b/web-console/src/utils/local-storage-backed-visibility.tsx
index c335180056b..f20031f2b8d 100644
--- a/web-console/src/utils/local-storage-backed-visibility.tsx
+++ b/web-console/src/utils/local-storage-backed-visibility.tsx
@@ -65,7 +65,7 @@ export class LocalStorageBackedVisibility {
return new LocalStorageBackedVisibility(this.key, defaultHidden, newVisibility);
}
- public shown(value: string): boolean {
- return this.visibility[value] ?? !this.defaultHidden.includes(value);
+ public shown(...values: string[]): boolean {
+ return values.some(value => this.visibility[value] ?? !this.defaultHidden.includes(value));
}
}
diff --git a/web-console/src/views/datasources-view/datasources-view.tsx b/web-console/src/views/datasources-view/datasources-view.tsx
index 75541b82999..997db1d91af 100644
--- a/web-console/src/views/datasources-view/datasources-view.tsx
+++ b/web-console/src/views/datasources-view/datasources-view.tsx
@@ -338,12 +338,11 @@ export class DatasourcesView extends React.PureComponent<
const columns = compact(
[
visibleColumns.shown('Datasource name') && `datasource`,
- (visibleColumns.shown('Availability') || visibleColumns.shown('Segment granularity')) && [
+ visibleColumns.shown('Availability', 'Segment granularity') && [
`COUNT(*) FILTER (WHERE is_active = 1) AS num_segments`,
`COUNT(*) FILTER (WHERE is_published = 1 AND is_overshadowed = 0 AND replication_factor = 0) AS num_zero_replica_segments`,
],
- (visibleColumns.shown('Availability') ||
- visibleColumns.shown('Historical load/drop queues')) && [
+ visibleColumns.shown('Availability', 'Historical load/drop queues') && [
`COUNT(*) FILTER (WHERE is_published = 1 AND is_overshadowed = 0 AND is_available = 0 AND replication_factor > 0) AS num_segments_to_load`,
`COUNT(*) FILTER (WHERE is_available = 1 AND is_active = 0) AS num_segments_to_drop`,
],
diff --git a/web-console/src/views/segments-view/segments-view.tsx b/web-console/src/views/segments-view/segments-view.tsx
index ba4b84b76c4..0e3fe1c6f4b 100644
--- a/web-console/src/views/segments-view/segments-view.tsx
+++ b/web-console/src/views/segments-view/segments-view.tsx
@@ -201,7 +201,7 @@ export class SegmentsView extends React.PureComponent<SegmentsViewProps, Segment
WHEN "start" LIKE '%:00.000Z' AND "end" LIKE '%:00.000Z' THEN 'Minute'
ELSE 'Sub minute'
END AS "time_span"`,
- (visibleColumns.shown('Shard type') || visibleColumns.shown('Shard spec')) && `"shard_spec"`,
+ visibleColumns.shown('Shard type', 'Shard spec') && `"shard_spec"`,
visibleColumns.shown('Partition') && `"partition_num"`,
visibleColumns.shown('Size') && `"size"`,
visibleColumns.shown('Num rows') && `"num_rows"`,
diff --git a/web-console/src/views/supervisors-view/supervisors-view.scss b/web-console/src/views/supervisors-view/supervisors-view.scss
index 7b03df4f186..25c55e69b44 100644
--- a/web-console/src/views/supervisors-view/supervisors-view.scss
+++ b/web-console/src/views/supervisors-view/supervisors-view.scss
@@ -37,5 +37,9 @@
font-style: italic;
opacity: 0.6;
}
+
+ .warning-line {
+ color: $orange4;
+ }
}
}
diff --git a/web-console/src/views/supervisors-view/supervisors-view.tsx b/web-console/src/views/supervisors-view/supervisors-view.tsx
index 6622cc99e11..316080641ce 100644
--- a/web-console/src/views/supervisors-view/supervisors-view.tsx
+++ b/web-console/src/views/supervisors-view/supervisors-view.tsx
@@ -92,6 +92,7 @@ const supervisorTableColumns: string[] = [
'Configured tasks',
'Running tasks',
'Aggregate lag',
+ 'Recent errors',
'Stats',
ACTION_COLUMN_LABEL,
];
@@ -102,6 +103,10 @@ function getRowStatsKeyTitle(key: RowStatsKey) {
return `Rate over past ${pluralIfNeeded(parseInt(key, 10), 'minute')}`;
}
+function getRowStatsKeySeconds(key: RowStatsKey): number {
+ return parseInt(key, 10) * 60;
+}
+
interface SupervisorQuery extends TableState {
capabilities: Capabilities;
visibleColumns: LocalStorageBackedVisibility;
@@ -288,7 +293,7 @@ export class SupervisorsView extends React.PureComponent<
}
if (capabilities.hasOverlordAccess()) {
- if (visibleColumns.shown('Running tasks') || visibleColumns.shown('Aggregate lag')) {
+ if (visibleColumns.shown('Running tasks', 'Aggregate lag', 'Recent errors')) {
try {
for (const supervisor of supervisors) {
cancelToken.throwIfRequested();
@@ -674,7 +679,7 @@ export class SupervisorsView extends React.PureComponent<
{
Header: 'Status',
id: 'detailed_state',
- width: 150,
+ width: 130,
accessor: 'detailed_state',
Cell: ({ value }) => (
<TableFilterableCell
@@ -758,7 +763,7 @@ export class SupervisorsView extends React.PureComponent<
},
{
Header: 'Aggregate lag',
- accessor: 'status.aggregateLag',
+ accessor: 'status.payload.aggregateLag',
width: 200,
filterable: false,
sortable: false,
@@ -800,23 +805,64 @@ export class SupervisorsView extends React.PureComponent<
Cell: ({ value }) => {
if (!value) return;
const c = getTotalSupervisorStats(value, statsKey);
- const isRate = statsKey !== 'totals';
- const formatNumber = isRate ? formatRate : formatInteger;
- const formatData = isRate ? formatByteRate : formatBytes;
- const bytes = c.processedBytes ? ` (${formatData(c.processedBytes)})` : '';
+ const seconds = getRowStatsKeySeconds(statsKey);
+ const totalLabel = `Total over ${statsKey}: `;
+ const bytes = c.processedBytes ? ` (${formatByteRate(c.processedBytes)})` : '';
return (
<div>
- <div>{`Processed: ${formatNumber(c.processed)}${bytes}`}</div>
+ <div
+ title={`${totalLabel}${formatInteger(c.processed * seconds)} (${formatBytes(
+ c.processedBytes * seconds,
+ )})`}
+ >{`Processed: ${formatRate(c.processed)}${bytes}`}</div>
{Boolean(c.processedWithError) && (
- <div>Processed with error: {formatNumber(c.processedWithError)}</div>
+ <div
+ className="warning-line"
+ title={`${totalLabel}${formatInteger(c.processedWithError * seconds)}`}
+ >
+ Processed with error: {formatRate(c.processedWithError)}
+ </div>
+ )}
+ {Boolean(c.thrownAway) && (
+ <div
+ className="warning-line"
+ title={`${totalLabel}${formatInteger(c.thrownAway * seconds)}`}
+ >
+ Thrown away: {formatRate(c.thrownAway)}
+ </div>
+ )}
+ {Boolean(c.unparseable) && (
+ <div
+ className="warning-line"
+ title={`${totalLabel}${formatInteger(c.unparseable * seconds)}`}
+ >
+ Unparseable: {formatRate(c.unparseable)}
+ </div>
)}
- {Boolean(c.thrownAway) && <div>Thrown away: {formatNumber(c.thrownAway)}</div>}
- {Boolean(c.unparseable) && <div>Unparseable: {formatNumber(c.unparseable)}</div>}
</div>
);
},
show: visibleColumns.shown('Stats'),
},
+ {
+ Header: 'Recent errors',
+ accessor: 'status.payload.recentErrors',
+ width: 200,
+ filterable: false,
+ sortable: false,
+ show: visibleColumns.shown('Recent errors'),
+ Cell: ({ value, original }) => {
+ return (
+ <TableClickableCell
+ onClick={() => this.onSupervisorDetail(original)}
+ hoverIcon={IconNames.SEARCH_TEMPLATE}
+ title="See errors"
+ >
+ {pluralIfNeeded(value?.length, 'error')}
+ </TableClickableCell>
+ );
+ },
+ },
{
Header: ACTION_COLUMN_LABEL,
id: ACTION_COLUMN_ID,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org
(druid) 03/04: detail
Posted by vo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch better_supervisor_view
in repository https://gitbox.apache.org/repos/asf/druid.git
commit bab61e1fd4fd2bea67821621b01a8f92a9fbdea5
Author: Vadim Ogievetsky <va...@ogievetsky.com>
AuthorDate: Fri Apr 19 13:50:27 2024 -0700
detail
---
web-console/src/views/supervisors-view/supervisors-view.scss | 5 +++++
web-console/src/views/supervisors-view/supervisors-view.tsx | 2 +-
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/web-console/src/views/supervisors-view/supervisors-view.scss b/web-console/src/views/supervisors-view/supervisors-view.scss
index 6db2b646d95..7b03df4f186 100644
--- a/web-console/src/views/supervisors-view/supervisors-view.scss
+++ b/web-console/src/views/supervisors-view/supervisors-view.scss
@@ -32,5 +32,10 @@
.title-button {
cursor: pointer;
}
+
+ .detail-line {
+ font-style: italic;
+ opacity: 0.6;
+ }
}
}
diff --git a/web-console/src/views/supervisors-view/supervisors-view.tsx b/web-console/src/views/supervisors-view/supervisors-view.tsx
index 0483cbd2184..6622cc99e11 100644
--- a/web-console/src/views/supervisors-view/supervisors-view.tsx
+++ b/web-console/src/views/supervisors-view/supervisors-view.tsx
@@ -707,7 +707,7 @@ export class SupervisorsView extends React.PureComponent<
return (
<div>
<div>{formatInteger(taskCount * replicas)}</div>
- <div>
+ <div className="detail-line">
{replicas === 1
? '(no replication)'
: `(${pluralIfNeeded(taskCount, 'task')} × ${pluralIfNeeded(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org