You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by cw...@apache.org on 2019/04/23 23:15:13 UTC
[incubator-druid] branch master updated: No SQL mode in web console
(#7493)
This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 11a7e91 No SQL mode in web console (#7493)
11a7e91 is described below
commit 11a7e91a73ec9412af11d669cf02cf7765b83ff2
Author: Qi Shu <sh...@gmail.com>
AuthorDate: Tue Apr 23 16:15:02 2019 -0700
No SQL mode in web console (#7493)
* Added no sql mode
* Use status code
* Add no sql mode to server view
* add sql broker check to decide if no sql mode should be enabled
* Fix historicals in home view
* Name change
* Add types for query result; improved functions
* Fixed a conflict/bug
* Fixed a bug
* multiple fix
* removed unused imports
* terminate query manager
* fix wording
---
web-console/src/console-application.tsx | 101 +++++++++++++++-------
web-console/src/variables.ts | 1 +
web-console/src/views/datasource-view.tsx | 54 +++++++++---
web-console/src/views/home-view.tsx | 92 ++++++++++++++------
web-console/src/views/segments-view.tsx | 138 +++++++++++++++++++++++++-----
web-console/src/views/servers-view.tsx | 73 +++++++++++++---
web-console/src/views/tasks-view.tsx | 70 ++++++++++++---
7 files changed, 416 insertions(+), 113 deletions(-)
diff --git a/web-console/src/console-application.tsx b/web-console/src/console-application.tsx
index 4d10638..4f5ea47 100644
--- a/web-console/src/console-application.tsx
+++ b/web-console/src/console-application.tsx
@@ -24,8 +24,10 @@ import * as React from 'react';
import { HashRouter, Route, Switch } from 'react-router-dom';
import { HeaderActiveTab, HeaderBar } from './components/header-bar';
+import {Loader} from './components/loader';
import { AppToaster } from './singletons/toaster';
-import { DRUID_DOCS_SQL, LEGACY_COORDINATOR_CONSOLE, LEGACY_OVERLORD_CONSOLE } from './variables';
+import {QueryManager} from './utils';
+import {DRUID_DOCS_API, DRUID_DOCS_SQL, LEGACY_COORDINATOR_CONSOLE, LEGACY_OVERLORD_CONSOLE} from './variables';
import { DatasourcesView } from './views/datasource-view';
import { HomeView } from './views/home-view';
import { LookupsView } from './views/lookups-view';
@@ -45,45 +47,54 @@ export interface ConsoleApplicationProps extends React.Props<any> {
export interface ConsoleApplicationState {
aboutDialogOpen: boolean;
+ noSqlMode: boolean;
+ capabilitiesLoading: boolean;
}
export class ConsoleApplication extends React.Component<ConsoleApplicationProps, ConsoleApplicationState> {
static MESSAGE_KEY = 'druid-console-message';
static MESSAGE_DISMISSED = 'dismissed';
+ private capabilitiesQueryManager: QueryManager<string, string>;
- static async ensureSql() {
+ static async discoverCapabilities(): Promise<'working-with-sql' | 'working-without-sql' | 'broken'> {
try {
await axios.post('/druid/v2/sql', { query: 'SELECT 1337' });
} catch (e) {
const { response } = e;
- if (response.status !== 405 || response.statusText !== 'Method Not Allowed') return true; // other failure
+ if (response.status !== 405 || response.statusText !== 'Method Not Allowed') return 'working-with-sql'; // other failure
try {
await axios.get('/status');
} catch (e) {
- return true; // total failure
+ return 'broken'; // total failure
}
-
// Status works but SQL 405s => the SQL endpoint is disabled
- AppToaster.show({
- icon: IconNames.ERROR,
- intent: Intent.DANGER,
- timeout: 120000,
- /* tslint:disable:jsx-alignment */
- message: <>
- It appears that the SQL endpoint is disabled. Either <a
- href={DRUID_DOCS_SQL}>enable the SQL endpoint</a> or use the old <a
- href={LEGACY_COORDINATOR_CONSOLE}>coordinator</a> and <a
- href={LEGACY_OVERLORD_CONSOLE}>overlord</a> consoles that do not rely on the SQL endpoint.
- </>
- /* tslint:enable:jsx-alignment */
- });
- return false;
+ return 'working-without-sql';
}
- return true;
+ return 'working-with-sql';
}
- static async shownNotifications() {
- await ConsoleApplication.ensureSql();
+ static shownNotifications(capabilities: string) {
+ let message: JSX.Element = <></>;
+ /* tslint:disable:jsx-alignment */
+ if (capabilities === 'working-without-sql') {
+ message = <>
+ It appears that the SQL endpoint is disabled. The console will fall back
+ to <a href={DRUID_DOCS_API} target="_blank">native Druid APIs</a> and will be
+ limited in functionality. Look at <a href={DRUID_DOCS_SQL} target="_blank">the SQL docs</a> to
+ enable the SQL endpoint.
+ </>;
+ } else if (capabilities === 'broken') {
+ message = <>
+ It appears that the Druid is not responding. Data cannot be retrieved right now
+ </>;
+ }
+ /* tslint:enable:jsx-alignment */
+ AppToaster.show({
+ icon: IconNames.ERROR,
+ intent: Intent.DANGER,
+ timeout: 120000,
+ message: message
+ });
}
private taskId: string | null;
@@ -95,7 +106,9 @@ export class ConsoleApplication extends React.Component<ConsoleApplicationProps,
constructor(props: ConsoleApplicationProps, context: any) {
super(props, context);
this.state = {
- aboutDialogOpen: false
+ aboutDialogOpen: false,
+ noSqlMode: false,
+ capabilitiesLoading: true
};
if (props.baseURL) {
@@ -104,10 +117,30 @@ export class ConsoleApplication extends React.Component<ConsoleApplicationProps,
if (props.customHeaderName && props.customHeaderValue) {
axios.defaults.headers.common[props.customHeaderName] = props.customHeaderValue;
}
+
+ this.capabilitiesQueryManager = new QueryManager({
+ processQuery: async (query: string) => {
+ const capabilities = await ConsoleApplication.discoverCapabilities();
+ if (capabilities !== 'working-with-sql') {
+ ConsoleApplication.shownNotifications(capabilities);
+ }
+ return capabilities;
+ },
+ onStateChange: ({ result, loading, error }) => {
+ this.setState({
+ noSqlMode: result === 'working-with-sql' ? false : true,
+ capabilitiesLoading: loading
+ });
+ }
+ });
}
componentDidMount(): void {
- ConsoleApplication.shownNotifications();
+ this.capabilitiesQueryManager.runQuery('dummy');
+ }
+
+ componentWillUnmount(): void {
+ this.capabilitiesQueryManager.terminate();
}
private resetInitialsDelay() {
@@ -147,6 +180,7 @@ export class ConsoleApplication extends React.Component<ConsoleApplicationProps,
render() {
const { hideLegacy } = this.props;
+ const { noSqlMode, capabilitiesLoading } = this.state;
const wrapInViewContainer = (active: HeaderActiveTab, el: JSX.Element, scrollable = false) => {
return <>
@@ -155,31 +189,40 @@ export class ConsoleApplication extends React.Component<ConsoleApplicationProps,
</>;
};
+ if (capabilitiesLoading) {
+ return <div className={'loading-capabilities'}>
+ <Loader
+ loadingText={''}
+ loading={capabilitiesLoading}
+ />
+ </div>;
+ }
+
return <HashRouter hashType="noslash">
<div className="console-application">
<Switch>
<Route
path="/datasources"
component={() => {
- return wrapInViewContainer('datasources', <DatasourcesView goToSql={this.goToSql} goToSegments={this.goToSegments}/>);
+ return wrapInViewContainer('datasources', <DatasourcesView goToSql={this.goToSql} goToSegments={this.goToSegments} noSqlMode={noSqlMode}/>);
}}
/>
<Route
path="/segments"
component={() => {
- return wrapInViewContainer('segments', <SegmentsView datasource={this.datasource} onlyUnavailable={this.onlyUnavailable} goToSql={this.goToSql}/>);
+ return wrapInViewContainer('segments', <SegmentsView datasource={this.datasource} onlyUnavailable={this.onlyUnavailable} goToSql={this.goToSql} noSqlMode={noSqlMode}/>);
}}
/>
<Route
path="/tasks"
component={() => {
- return wrapInViewContainer('tasks', <TasksView taskId={this.taskId} goToSql={this.goToSql} goToMiddleManager={this.goToMiddleManager}/>, true);
+ return wrapInViewContainer('tasks', <TasksView taskId={this.taskId} goToSql={this.goToSql} goToMiddleManager={this.goToMiddleManager} noSqlMode={noSqlMode}/>, true);
}}
/>
<Route
path="/servers"
component={() => {
- return wrapInViewContainer('servers', <ServersView middleManager={this.middleManager} goToSql={this.goToSql} goToTask={this.goToTask}/>, true);
+ return wrapInViewContainer('servers', <ServersView middleManager={this.middleManager} goToSql={this.goToSql} goToTask={this.goToTask} noSqlMode={noSqlMode}/>, true);
}}
/>
<Route
@@ -196,7 +239,7 @@ export class ConsoleApplication extends React.Component<ConsoleApplicationProps,
/>
<Route
component={() => {
- return wrapInViewContainer(null, <HomeView/>);
+ return wrapInViewContainer(null, <HomeView noSqlMode={noSqlMode}/>);
}}
/>
</Switch>
diff --git a/web-console/src/variables.ts b/web-console/src/variables.ts
index b338bc2..10a51ec 100644
--- a/web-console/src/variables.ts
+++ b/web-console/src/variables.ts
@@ -26,3 +26,4 @@ export const DRUID_DOCS_SQL = 'http://druid.io/docs/latest/querying/sql.html';
export const DRUID_COMMUNITY = 'http://druid.io/community/';
export const DRUID_USER_GROUP = 'https://groups.google.com/forum/#!forum/druid-user';
export const DRUID_DEVELOPER_GROUP = 'https://lists.apache.org/list.html?dev@druid.apache.org';
+export const DRUID_DOCS_API = 'http://druid.io/docs/latest/operations/api-reference.html';
diff --git a/web-console/src/views/datasource-view.tsx b/web-console/src/views/datasource-view.tsx
index a9061a0..406599a 100644
--- a/web-console/src/views/datasource-view.tsx
+++ b/web-console/src/views/datasource-view.tsx
@@ -44,10 +44,12 @@ import {
import './datasource-view.scss';
const tableColumns: string[] = ['Datasource', 'Availability', 'Retention', 'Compaction', 'Size', 'Num rows', 'Actions'];
+const tableColumnsNoSql: string[] = ['Datasource', 'Availability', 'Retention', 'Compaction', 'Size', 'Actions'];
export interface DatasourcesViewProps extends React.Props<any> {
goToSql: (initSql: string) => void;
goToSegments: (datasource: string, onlyUnavailable?: boolean) => void;
+ noSqlMode: boolean;
}
interface Datasource {
@@ -56,6 +58,14 @@ interface Datasource {
[key: string]: any;
}
+interface DatasourceQueryResultRow {
+ datasource: string;
+ num_available_segments: number;
+ num_rows: number;
+ num_segments: number;
+ size: number;
+}
+
export interface DatasourcesViewState {
datasourcesLoading: boolean;
datasources: Datasource[] | null;
@@ -116,9 +126,28 @@ export class DatasourcesView extends React.Component<DatasourcesViewProps, Datas
}
componentDidMount(): void {
+ const { noSqlMode } = this.props;
+
this.datasourceQueryManager = new QueryManager({
processQuery: async (query: string) => {
- const datasources: any[] = await queryDruidSql({ query });
+ let datasources: DatasourceQueryResultRow[];
+ if (!noSqlMode) {
+ datasources = await queryDruidSql({ query });
+ } else {
+ const datasourcesResp = await axios.get('/druid/coordinator/v1/datasources?simple');
+ const loadstatusResp = await axios.get('/druid/coordinator/v1/loadstatus?simple');
+ const loadstatus = loadstatusResp.data;
+ datasources = datasourcesResp.data.map((d: any) => {
+ return {
+ datasource: d.name,
+ num_available_segments: d.properties.segments.count,
+ size: d.properties.segments.size,
+ num_segments: d.properties.segments.count + loadstatus[d.name],
+ num_rows: -1
+ };
+ });
+ }
+
const seen = countBy(datasources, (x: any) => x.datasource);
const disabledResp = await axios.get('/druid/coordinator/v1/metadata/datasources?includeDisabled');
@@ -133,7 +162,7 @@ export class DatasourcesView extends React.Component<DatasourcesViewProps, Datas
const tiersResp = await axios.get('/druid/coordinator/v1/tiers');
const tiers = tiersResp.data;
- const allDatasources = datasources.concat(disabled.map(d => ({ datasource: d, disabled: true })));
+ const allDatasources = (datasources as any).concat(disabled.map(d => ({ datasource: d, disabled: true })));
allDatasources.forEach((ds: any) => {
ds.rules = rules[ds.datasource] || [];
ds.compaction = compaction[ds.datasource];
@@ -354,7 +383,7 @@ GROUP BY 1`);
}
renderDatasourceTable() {
- const { goToSegments } = this.props;
+ const { goToSegments, noSqlMode } = this.props;
const { datasources, defaultRules, datasourcesLoading, datasourcesError, datasourcesFilter, showDisabled } = this.state;
const { tableColumnSelectionHandler } = this;
let data = datasources || [];
@@ -492,7 +521,7 @@ GROUP BY 1`);
filterable: false,
width: 100,
Cell: (row) => formatNumber(row.value),
- show: tableColumnSelectionHandler.showColumn('Num rows')
+ show: !noSqlMode && tableColumnSelectionHandler.showColumn('Num rows')
},
{
Header: 'Actions',
@@ -529,7 +558,7 @@ GROUP BY 1`);
}
render() {
- const { goToSql } = this.props;
+ const { goToSql, noSqlMode } = this.props;
const { showDisabled } = this.state;
const { tableColumnSelectionHandler } = this;
@@ -540,18 +569,21 @@ GROUP BY 1`);
text="Refresh"
onClick={() => this.datasourceQueryManager.rerunLastQuery()}
/>
- <Button
- icon={IconNames.APPLICATION}
- text="Go to SQL"
- onClick={() => goToSql(this.datasourceQueryManager.getLastQuery())}
- />
+ {
+ !noSqlMode &&
+ <Button
+ icon={IconNames.APPLICATION}
+ text="Go to SQL"
+ onClick={() => goToSql(this.datasourceQueryManager.getLastQuery())}
+ />
+ }
<Switch
checked={showDisabled}
label="Show disabled"
onChange={() => this.setState({ showDisabled: !showDisabled })}
/>
<TableColumnSelection
- columns={tableColumns}
+ columns={noSqlMode ? tableColumnsNoSql : tableColumns}
onChange={(column) => tableColumnSelectionHandler.changeTableColumnSelection(column)}
tableColumnsHidden={tableColumnSelectionHandler.hiddenColumns}
/>
diff --git a/web-console/src/views/home-view.tsx b/web-console/src/views/home-view.tsx
index 303096e..33eb6e3 100644
--- a/web-console/src/views/home-view.tsx
+++ b/web-console/src/views/home-view.tsx
@@ -19,7 +19,6 @@
import { Card, H5, Icon } from '@blueprintjs/core';
import { IconName, IconNames } from '@blueprintjs/icons';
import axios from 'axios';
-import * as classNames from 'classnames';
import * as React from 'react';
import { getHeadProp, pluralIfNeeded, queryDruidSql, QueryManager } from '../utils';
@@ -36,6 +35,7 @@ export interface CardOptions {
}
export interface HomeViewProps extends React.Props<any> {
+ noSqlMode: boolean;
}
export interface HomeViewState {
@@ -110,6 +110,8 @@ export class HomeView extends React.Component<HomeViewProps, HomeViewState> {
}
componentDidMount(): void {
+ const { noSqlMode } = this.props;
+
this.statusQueryManager = new QueryManager({
processQuery: async (query) => {
const statusResp = await axios.get('/status');
@@ -130,7 +132,13 @@ export class HomeView extends React.Component<HomeViewProps, HomeViewState> {
this.datasourceQueryManager = new QueryManager({
processQuery: async (query) => {
- const datasources = await queryDruidSql({ query });
+ let datasources: string[];
+ if (!noSqlMode) {
+ datasources = await queryDruidSql({ query });
+ } else {
+ const datasourcesResp = await axios.get('/druid/coordinator/v1/datasources');
+ datasources = datasourcesResp.data;
+ }
return datasources.length;
},
onStateChange: ({ result, loading, error }) => {
@@ -148,8 +156,26 @@ export class HomeView extends React.Component<HomeViewProps, HomeViewState> {
this.segmentQueryManager = new QueryManager({
processQuery: async (query) => {
- const segments = await queryDruidSql({ query });
- return getHeadProp(segments, 'count') || 0;
+ if (!noSqlMode) {
+ const segments = await queryDruidSql({ query });
+ return getHeadProp(segments, 'count') || 0;
+ } else {
+
+ const loadstatusResp = await axios.get('/druid/coordinator/v1/loadstatus?simple');
+ const loadstatus = loadstatusResp.data;
+ const unavailableSegmentNum = Object.keys(loadstatus).reduce((sum, key) => {
+ return sum + loadstatus[key];
+ }, 0);
+
+ const datasourcesMetaResp = await axios.get('/druid/coordinator/v1/datasources?simple');
+ const datasourcesMeta = datasourcesMetaResp.data;
+ const availableSegmentNum = datasourcesMeta.reduce((sum: number, curr: any) => {
+ return sum + curr.properties.segments.count;
+ }, 0);
+
+ return availableSegmentNum + unavailableSegmentNum;
+ }
+
},
onStateChange: ({ result, loading, error }) => {
this.setState({
@@ -166,27 +192,27 @@ export class HomeView extends React.Component<HomeViewProps, HomeViewState> {
this.taskQueryManager = new QueryManager({
processQuery: async (query) => {
- const taskCountsFromSql = await queryDruidSql({ query });
- const taskCounts = {
- successTaskCount: 0,
- failedTaskCount: 0,
- runningTaskCount: 0,
- waitingTaskCount: 0,
- pendingTaskCount: 0
- };
- for (const dataStatus of taskCountsFromSql) {
- if (dataStatus.status === 'SUCCESS') {
- taskCounts.successTaskCount = dataStatus.count;
- } else if (dataStatus.status === 'FAILED') {
- taskCounts.failedTaskCount = dataStatus.count;
- } else if (dataStatus.status === 'RUNNING') {
- taskCounts.runningTaskCount = dataStatus.count;
- } else if (dataStatus.status === 'WAITING') {
- taskCounts.waitingTaskCount = dataStatus.count;
- } else {
- taskCounts.pendingTaskCount = dataStatus.count;
- }
+ let taskCountsFromQuery: {status: string, count: number}[] = [];
+ if (!noSqlMode) {
+ taskCountsFromQuery = await queryDruidSql({ query });
+ } else {
+ const completeTasksResp = await axios.get('/druid/indexer/v1/completeTasks');
+ const runningTasksResp = await axios.get('/druid/indexer/v1/runningTasks');
+ const waitingTasksResp = await axios.get('/druid/indexer/v1/waitingTasks');
+ const pendingTasksResp = await axios.get('/druid/indexer/v1/pendingTasks');
+ taskCountsFromQuery.push(
+ {status: 'SUCCESS', count: completeTasksResp.data.filter((d: any) => d.status === 'SUCCESS').length},
+ {status: 'FAILED', count: completeTasksResp.data.filter((d: any) => d.status === 'FAILED').length},
+ {status: 'RUNNING', count: runningTasksResp.data.length},
+ {status: 'WAITING', count: waitingTasksResp.data.length},
+ {status: 'PENDING', count: pendingTasksResp.data.length}
+ );
}
+ const taskCounts = taskCountsFromQuery.reduce((acc: any, curr: any) => {
+ const status = curr.status.toLowerCase();
+ const property = `${status}TaskCount`;
+ return {...acc, [property]: curr.count};
+ }, {});
return taskCounts;
},
onStateChange: ({ result, loading, error }) => {
@@ -212,8 +238,19 @@ GROUP BY 1`);
this.dataServerQueryManager = new QueryManager({
processQuery: async (query) => {
- const dataServerCounts = await queryDruidSql({ query });
- return getHeadProp(dataServerCounts, 'count') || 0;
+ const getDataServerNum = async () => {
+ const allServerResp = await axios.get('/druid/coordinator/v1/servers?simple');
+ const allServers = allServerResp.data;
+ return allServers.filter((s: any) => s.type === 'historical').length;
+ };
+ if (!noSqlMode) {
+ const dataServerCounts = await queryDruidSql({ query });
+ const serverNum = getHeadProp(dataServerCounts, 'count') || 0;
+ if (serverNum === 0) return await getDataServerNum();
+ return serverNum;
+ } else {
+ return await getDataServerNum();
+ }
},
onStateChange: ({ result, loading, error }) => {
this.setState({
@@ -306,7 +343,8 @@ GROUP BY 1`);
{Boolean(state.successTaskCount) && <p>{pluralIfNeeded(state.successTaskCount, 'successful task')}</p>}
{Boolean(state.waitingTaskCount) && <p>{pluralIfNeeded(state.waitingTaskCount, 'waiting task')}</p>}
{Boolean(state.failedTaskCount) && <p>{pluralIfNeeded(state.failedTaskCount, 'failed task')}</p>}
- { !(state.runningTaskCount + state.pendingTaskCount + state.successTaskCount + state.waitingTaskCount + state.failedTaskCount) &&
+ {!(Boolean(state.runningTaskCount) || Boolean(state.pendingTaskCount) || Boolean(state.successTaskCount) ||
+ Boolean(state.waitingTaskCount) || Boolean(state.failedTaskCount)) &&
<p>There are no tasks</p>
}
</>,
diff --git a/web-console/src/views/segments-view.tsx b/web-console/src/views/segments-view.tsx
index c150fac..1dd3b8a 100644
--- a/web-console/src/views/segments-view.tsx
+++ b/web-console/src/views/segments-view.tsx
@@ -20,14 +20,12 @@ import { Button, Intent } from '@blueprintjs/core';
import { H5 } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import axios from 'axios';
-import * as classNames from 'classnames';
import * as React from 'react';
import ReactTable from 'react-table';
import { Filter } from 'react-table';
import { TableColumnSelection } from '../components/table-column-selection';
import { ViewControlBar } from '../components/view-control-bar';
-import { AppToaster } from '../singletons/toaster';
import {
addFilter,
formatBytes,
@@ -44,18 +42,21 @@ import './segments-view.scss';
const tableColumns: string[] = ['Segment ID', 'Datasource', 'Start', 'End', 'Version', 'Partition',
'Size', 'Num rows', 'Replicas', 'Is published', 'Is realtime', 'Is available'];
+const tableColumnsNoSql: string[] = ['Segment ID', 'Datasource', 'Start', 'End', 'Version', 'Partition', 'Size'];
export interface SegmentsViewProps extends React.Props<any> {
goToSql: (initSql: string) => void;
datasource: string | null;
onlyUnavailable: boolean | null;
+ noSqlMode: boolean;
}
export interface SegmentsViewState {
segmentsLoading: boolean;
- segments: any[] | null;
+ segments: SegmentQueryResultRow[] | null;
segmentsError: string | null;
segmentFilter: Filter[];
+ allSegments?: SegmentQueryResultRow[] | null;
}
interface QueryAndSkip {
@@ -63,8 +64,25 @@ interface QueryAndSkip {
skip: number;
}
+interface SegmentQueryResultRow {
+ datasource: string;
+ start: string;
+ end: string;
+ segment_id: string;
+ version: string;
+ size: 0;
+ partition_num: number;
+ payload: any;
+ num_rows: number;
+ num_replicas: number;
+ is_available: number;
+ is_published: number;
+ is_realtime: number;
+}
+
export class SegmentsView extends React.Component<SegmentsViewProps, SegmentsViewState> {
- private segmentsQueryManager: QueryManager<QueryAndSkip, any[]>;
+ private segmentsSqlQueryManager: QueryManager<QueryAndSkip, SegmentQueryResultRow[]>;
+ private segmentsJsonQueryManager: QueryManager<any, SegmentQueryResultRow[]>;
private tableColumnSelectionHandler: TableColumnSelectionHandler;
constructor(props: SegmentsViewProps, context: any) {
@@ -81,7 +99,7 @@ export class SegmentsView extends React.Component<SegmentsViewProps, SegmentsVie
segmentFilter
};
- this.segmentsQueryManager = new QueryManager({
+ this.segmentsSqlQueryManager = new QueryManager({
processQuery: async (query: QueryAndSkip) => {
const results: any[] = (await queryDruidSql({ query: query.query })).slice(query.skip);
results.forEach(result => {
@@ -102,13 +120,58 @@ export class SegmentsView extends React.Component<SegmentsViewProps, SegmentsVie
}
});
+ this.segmentsJsonQueryManager = new QueryManager({
+ processQuery: async (query: any) => {
+ const datasourceList = (await axios.get('/druid/coordinator/v1/metadata/datasources')).data;
+ const nestedResults: SegmentQueryResultRow[][] = await Promise.all(datasourceList.map(async (d: string) => {
+ const segments = (await axios.get(`/druid/coordinator/v1/datasources/${d}?full`)).data.segments;
+ return segments.map((segment: any) => {
+ return {
+ segment_id: segment.identifier,
+ datasource: segment.dataSource,
+ start: segment.interval.split('/')[0],
+ end: segment.interval.split('/')[1],
+ version: segment.version,
+ partition_num: segment.shardSpec.partitionNum ? 0 : segment.shardSpec.partitionNum,
+ size: segment.size,
+ payload: segment,
+ num_rows: -1,
+ num_replicas: -1,
+ is_available: -1,
+ is_published: -1,
+ is_realtime: -1
+ };
+ });
+ }));
+ const results: SegmentQueryResultRow[] = [].concat.apply([], nestedResults).sort((d1: any, d2: any) => {
+ return d2.start.localeCompare(d1.start);
+ });
+ return results;
+ },
+ onStateChange: ({ result, loading, error }) => {
+ this.setState({
+ allSegments: result,
+ segments: result ? result.slice(0, 50) : null,
+ segmentsLoading: loading,
+ segmentsError: error
+ });
+ }
+ });
+
this.tableColumnSelectionHandler = new TableColumnSelectionHandler(
LocalStorageKeys.SEGMENT_TABLE_COLUMN_SELECTION, () => this.setState({})
);
}
+ componentDidMount(): void {
+ if (this.props.noSqlMode) {
+ this.segmentsJsonQueryManager.runQuery('init');
+ }
+ }
+
componentWillUnmount(): void {
- this.segmentsQueryManager.terminate();
+ this.segmentsSqlQueryManager.terminate();
+ this.segmentsJsonQueryManager.terminate();
}
private fetchData = (state: any, instance: any) => {
@@ -140,15 +203,42 @@ export class SegmentsView extends React.Component<SegmentsViewProps, SegmentsVie
queryParts.push(`LIMIT ${totalQuerySize}`);
const query = queryParts.join('\n');
-
- this.segmentsQueryManager.runQuery({
+ this.segmentsSqlQueryManager.runQuery({
query,
skip: totalQuerySize - pageSize
});
}
+ private fecthClientSideData = (state: any, instance: any) => {
+ const { page, pageSize, filtered, sorted } = state;
+ const { allSegments } = this.state;
+ if (allSegments == null) return;
+ const startPage = page * pageSize;
+ const endPage = (page + 1) * pageSize;
+ const sortPivot = sorted[0].id;
+ const sortDesc = sorted[0].desc;
+ const selectedSegments = allSegments.sort((d1: any, d2: any) => {
+ const v1 = d1[sortPivot];
+ const v2 = d2[sortPivot];
+ if (typeof (d1[sortPivot]) === 'string') {
+ return sortDesc ? v2.localeCompare(v1) : v1.localeCompare(v2);
+ } else {
+ return sortDesc ? v2 - v1 : v1 - v2;
+ }
+ }).filter((d: any) => {
+ return filtered.every((f: any) => {
+ return d[f.id].includes(f.value);
+ });
+ });
+ const segments = selectedSegments.slice(startPage, endPage);
+ this.setState({
+ segments
+ });
+ }
+
renderSegmentsTable() {
const { segments, segmentsLoading, segmentsError, segmentFilter } = this.state;
+ const { noSqlMode } = this.props;
const { tableColumnSelectionHandler } = this;
return <ReactTable
@@ -163,7 +253,7 @@ export class SegmentsView extends React.Component<SegmentsViewProps, SegmentsVie
onFilteredChange={(filtered, column) => {
this.setState({ segmentFilter: filtered });
}}
- onFetchData={this.fetchData}
+ onFetchData={noSqlMode ? this.fecthClientSideData : this.fetchData}
showPageJump={false}
ofText=""
columns={[
@@ -232,7 +322,7 @@ export class SegmentsView extends React.Component<SegmentsViewProps, SegmentsVie
filterable: false,
defaultSortDesc: true,
Cell: row => formatNumber(row.value),
- show: tableColumnSelectionHandler.showColumn('Num rows')
+ show: !noSqlMode && tableColumnSelectionHandler.showColumn('Num rows')
},
{
Header: 'Replicas',
@@ -240,28 +330,28 @@ export class SegmentsView extends React.Component<SegmentsViewProps, SegmentsVie
width: 60,
filterable: false,
defaultSortDesc: true,
- show: tableColumnSelectionHandler.showColumn('Replicas')
+ show: !noSqlMode && tableColumnSelectionHandler.showColumn('Replicas')
},
{
Header: 'Is published',
id: 'is_published',
accessor: (row) => String(Boolean(row.is_published)),
Filter: makeBooleanFilter(),
- show: tableColumnSelectionHandler.showColumn('Is published')
+ show: !noSqlMode && tableColumnSelectionHandler.showColumn('Is published')
},
{
Header: 'Is realtime',
id: 'is_realtime',
accessor: (row) => String(Boolean(row.is_realtime)),
Filter: makeBooleanFilter(),
- show: tableColumnSelectionHandler.showColumn('Is realtime')
+ show: !noSqlMode && tableColumnSelectionHandler.showColumn('Is realtime')
},
{
Header: 'Is available',
id: 'is_available',
accessor: (row) => String(Boolean(row.is_available)),
Filter: makeBooleanFilter(),
- show: tableColumnSelectionHandler.showColumn('Is available')
+ show: !noSqlMode && tableColumnSelectionHandler.showColumn('Is available')
}
]}
defaultPageSize={50}
@@ -284,7 +374,7 @@ export class SegmentsView extends React.Component<SegmentsViewProps, SegmentsVie
}
render() {
- const { goToSql } = this.props;
+ const { goToSql, noSqlMode } = this.props;
const { tableColumnSelectionHandler } = this;
return <div className="segments-view app-view">
@@ -292,15 +382,19 @@ export class SegmentsView extends React.Component<SegmentsViewProps, SegmentsVie
<Button
icon={IconNames.REFRESH}
text="Refresh"
- onClick={() => this.segmentsQueryManager.rerunLastQuery()}
- />
- <Button
- icon={IconNames.APPLICATION}
- text="Go to SQL"
- onClick={() => goToSql(this.segmentsQueryManager.getLastQuery().query)}
+ onClick={() => noSqlMode ? this.segmentsJsonQueryManager.rerunLastQuery() : this.segmentsSqlQueryManager.rerunLastQuery()}
/>
+ {
+ !noSqlMode &&
+ <Button
+ icon={IconNames.APPLICATION}
+ text="Go to SQL"
+ hidden={noSqlMode}
+ onClick={() => goToSql(this.segmentsSqlQueryManager.getLastQuery().query)}
+ />
+ }
<TableColumnSelection
- columns={tableColumns}
+ columns={noSqlMode ? tableColumnsNoSql : tableColumns}
onChange={(column) => tableColumnSelectionHandler.changeTableColumnSelection(column)}
tableColumnsHidden={tableColumnSelectionHandler.hiddenColumns}
/>
diff --git a/web-console/src/views/servers-view.tsx b/web-console/src/views/servers-view.tsx
index f04ed44..832cebd 100644
--- a/web-console/src/views/servers-view.tsx
+++ b/web-console/src/views/servers-view.tsx
@@ -19,7 +19,6 @@
import { Button, Switch } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import axios from 'axios';
-import * as classNames from 'classnames';
import { sum } from 'd3-array';
import * as React from 'react';
import ReactTable from 'react-table';
@@ -55,6 +54,7 @@ export interface ServersViewProps extends React.Props<any> {
middleManager: string | null;
goToSql: (initSql: string) => void;
goToTask: (taskId: string) => void;
+ noSqlMode: boolean;
}
export interface ServersViewState {
@@ -70,9 +70,32 @@ export interface ServersViewState {
middleManagerFilter: Filter[];
}
+interface ServerQueryResultRow {
+ curr_size: number;
+ host: string;
+ max_size: number;
+ plaintext_port: number;
+ server: string;
+ tier: string;
+ tls_port: number;
+ segmentsToDrop?: number;
+ segmentsToDropSize?: number;
+ segmentsToLoad?: number;
+ segmentsToLoadSize?: number;
+}
+
+interface MiddleManagerQueryResultRow {
+ availabilityGroups: string[];
+ blacklistedUntil: string | null;
+ currCapacityUsed: number;
+ lastCompletedTaskTime: string;
+ runningTasks: string[];
+ worker: any;
+}
+
export class ServersView extends React.Component<ServersViewProps, ServersViewState> {
- private serverQueryManager: QueryManager<string, any[]>;
- private middleManagerQueryManager: QueryManager<string, any[]>;
+ private serverQueryManager: QueryManager<string, ServerQueryResultRow[]>;
+ private middleManagerQueryManager: QueryManager<string, MiddleManagerQueryResultRow[]>;
private serverTableColumnSelectionHandler: TableColumnSelectionHandler;
private middleManagerTableColumnSelectionHandler: TableColumnSelectionHandler;
@@ -100,14 +123,37 @@ export class ServersView extends React.Component<ServersViewProps, ServersViewSt
);
}
+ static getServers = async (): Promise<ServerQueryResultRow[]> => {
+ const allServerResp = await axios.get('/druid/coordinator/v1/servers?simple');
+ const allServers = allServerResp.data;
+ return allServers.filter((s: any) => s.type === 'historical').map((s: any) => {
+ return {
+ host: s.host.split(':')[0],
+ plaintext_port: parseInt(s.host.split(':')[1], 10),
+ server: s.host,
+ curr_size: s.currSize,
+ max_size: s.maxSize,
+ tier: s.tier,
+ tls_port: -1
+ };
+ });
+ }
+
componentDidMount(): void {
+ const { noSqlMode } = this.props;
this.serverQueryManager = new QueryManager({
processQuery: async (query: string) => {
- const servers = await queryDruidSql({ query });
-
+ let servers: ServerQueryResultRow[];
+ if (!noSqlMode) {
+ servers = await queryDruidSql({ query });
+ if (servers.length === 0) {
+ servers = await ServersView.getServers();
+ }
+ } else {
+ servers = await ServersView.getServers();
+ }
const loadQueueResponse = await axios.get('/druid/coordinator/v1/loadqueue?simple');
const loadQueues = loadQueueResponse.data;
-
return servers.map((s: any) => {
const loadQueueInfo = loadQueues[s.server];
if (loadQueueInfo) {
@@ -366,7 +412,7 @@ WHERE "server_type" = 'historical'`);
}
render() {
- const { goToSql } = this.props;
+ const { goToSql, noSqlMode } = this.props;
const { groupByTier } = this.state;
const { serverTableColumnSelectionHandler, middleManagerTableColumnSelectionHandler } = this;
@@ -377,11 +423,14 @@ WHERE "server_type" = 'historical'`);
text="Refresh"
onClick={() => this.serverQueryManager.rerunLastQuery()}
/>
- <Button
- icon={IconNames.APPLICATION}
- text="Go to SQL"
- onClick={() => goToSql(this.serverQueryManager.getLastQuery())}
- />
+ {
+ !noSqlMode &&
+ <Button
+ icon={IconNames.APPLICATION}
+ text="Go to SQL"
+ onClick={() => goToSql(this.serverQueryManager.getLastQuery())}
+ />
+ }
<Switch
checked={groupByTier}
label="Group by tier"
diff --git a/web-console/src/views/tasks-view.tsx b/web-console/src/views/tasks-view.tsx
index 0de77a4..6f11c87 100644
--- a/web-console/src/views/tasks-view.tsx
+++ b/web-console/src/views/tasks-view.tsx
@@ -19,7 +19,6 @@
import { Alert, Button, ButtonGroup, Intent, Label } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import axios from 'axios';
-import * as classNames from 'classnames';
import * as React from 'react';
import ReactTable from 'react-table';
import { Filter } from 'react-table';
@@ -48,6 +47,7 @@ export interface TasksViewProps extends React.Props<any> {
taskId: string | null;
goToSql: (initSql: string) => void;
goToMiddleManager: (middleManager: string) => void;
+ noSqlMode: boolean;
}
export interface TasksViewState {
@@ -73,6 +73,23 @@ export interface TasksViewState {
alertErrorMsg: string | null;
}
+interface TaskQueryResultRow {
+ created_time: string;
+ datasource: string;
+ duration: number;
+ error_msg: string | null;
+ location: string | null;
+ rank: number;
+ status: string;
+ task_id: string;
+ type: string;
+}
+
+interface SupervisorQueryResultRow {
+ id: string;
+ spec: any;
+}
+
function statusToColor(status: string): string {
switch (status) {
case 'RUNNING': return '#2167d5';
@@ -85,11 +102,11 @@ function statusToColor(status: string): string {
}
export class TasksView extends React.Component<TasksViewProps, TasksViewState> {
- private supervisorQueryManager: QueryManager<string, any[]>;
- private taskQueryManager: QueryManager<string, any[]>;
+ private supervisorQueryManager: QueryManager<string, SupervisorQueryResultRow[]>;
+ private taskQueryManager: QueryManager<string, TaskQueryResultRow[]>;
private supervisorTableColumnSelectionHandler: TableColumnSelectionHandler;
private taskTableColumnSelectionHandler: TableColumnSelectionHandler;
- private statusRanking = {RUNNING: 4, PENDING: 3, WAITING: 2, SUCCESS: 1, FAILED: 1};
+ static statusRanking = {RUNNING: 4, PENDING: 3, WAITING: 2, SUCCESS: 1, FAILED: 1};
constructor(props: TasksViewProps, context: any) {
super(props, context);
@@ -126,7 +143,24 @@ export class TasksView extends React.Component<TasksViewProps, TasksViewState> {
);
}
+ static parseTasks = (data: any[]): TaskQueryResultRow[] => {
+ return data.map((d: any) => {
+ return {
+ created_time: d.createdTime,
+ datasource: d.dataSource,
+ duration: d.duration ? d.duration : 0,
+ error_msg: d.errorMsg,
+ location: d.location.host ? `${d.location.host}:${d.location.port}` : null,
+ rank: (TasksView.statusRanking as any)[d.statusCode === 'RUNNING' ? d.runnerStatusCode : d.statusCode],
+ status: d.statusCode === 'RUNNING' ? d.runnerStatusCode : d.statusCode,
+ task_id: d.id,
+ type: d.typTasksView
+ };
+ });
+ }
+
componentDidMount(): void {
+ const { noSqlMode } = this.props;
this.supervisorQueryManager = new QueryManager({
processQuery: async (query: string) => {
const resp = await axios.get('/druid/indexer/v1/supervisor?full');
@@ -145,7 +179,16 @@ export class TasksView extends React.Component<TasksViewProps, TasksViewState> {
this.taskQueryManager = new QueryManager({
processQuery: async (query: string) => {
- return await queryDruidSql({ query });
+ if (!noSqlMode) {
+ return await queryDruidSql({ query });
+ } else {
+ const taskEndpoints: string[] = ['completeTasks', 'runningTasks', 'waitingTasks', 'pendingTasks'];
+ const result: TaskQueryResultRow[][] = await Promise.all(taskEndpoints.map(async (endpoint: string) => {
+ const resp = await axios.get(`/druid/indexer/v1/${endpoint}`);
+ return TasksView.parseTasks(resp.data);
+ }));
+ return [].concat.apply([], result);
+ }
},
onStateChange: ({ result, loading, error }) => {
this.setState({
@@ -523,7 +566,7 @@ ORDER BY "rank" DESC, "created_time" DESC`);
return <span>{Object.keys(previewCount).sort().map(v => `${v} (${previewCount[v]})`).join(', ')}</span>;
},
sortMethod: (d1, d2) => {
- const statusRanking: any = this.statusRanking;
+ const statusRanking: any = TasksView.statusRanking;
return statusRanking[d1.status] - statusRanking[d2.status] || d1.created_time.localeCompare(d2.created_time);
},
filterMethod: (filter: Filter, row: any) => {
@@ -570,7 +613,7 @@ ORDER BY "rank" DESC, "created_time" DESC`);
}
render() {
- const { goToSql } = this.props;
+ const { goToSql, noSqlMode } = this.props;
const { groupTasksBy, supervisorSpecDialogOpen, taskSpecDialogOpen, alertErrorMsg } = this.state;
const { supervisorTableColumnSelectionHandler, taskTableColumnSelectionHandler } = this;
@@ -609,11 +652,14 @@ ORDER BY "rank" DESC, "created_time" DESC`);
text="Refresh"
onClick={() => this.taskQueryManager.rerunLastQuery()}
/>
- <Button
- icon={IconNames.APPLICATION}
- text="Go to SQL"
- onClick={() => goToSql(this.taskQueryManager.getLastQuery())}
- />
+ {
+ !noSqlMode &&
+ <Button
+ icon={IconNames.APPLICATION}
+ text="Go to SQL"
+ onClick={() => goToSql(this.taskQueryManager.getLastQuery())}
+ />
+ }
<Button
icon={IconNames.PLUS}
text="Submit task"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org