You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by vo...@apache.org on 2023/04/20 17:26:50 UTC
[druid] branch master updated: Web console: better end of (MSQ) query segment loading UX (#14120)
This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new e7ae825e0c Web console: better end of (MSQ) query segment loading UX (#14120)
e7ae825e0c is described below
commit e7ae825e0cdd5a6b6726df504d870ac66a090438
Author: Vadim Ogievetsky <va...@ogievetsky.com>
AuthorDate: Thu Apr 20 10:26:43 2023 -0700
Web console: better end of (MSQ) query segment loading UX (#14120)
* better end of query segment loading UX
* fix snapshot
* handle case when MSQ query returns results directly
* add ip address column icon
* better icons
* add variance icon
* better summary
---
.../record-table-pane/record-table-pane.tsx | 3 +-
.../src/druid-models/execution/execution.ts | 8 +--
web-console/src/helpers/execution/general.ts | 4 +-
.../src/helpers/execution/sql-task-execution.ts | 59 ++++++++++++++--------
web-console/src/utils/types.ts | 24 +++++++--
.../schema-step/preview-table/preview-table.tsx | 10 +++-
.../__snapshots__/column-tree.spec.tsx.snap | 2 +-
.../execution-progress-bar-pane.spec.tsx.snap | 2 +-
.../execution-progress-bar-pane.tsx | 2 +-
.../result-table-pane/result-table-pane.tsx | 3 +-
10 files changed, 79 insertions(+), 38 deletions(-)
diff --git a/web-console/src/components/record-table-pane/record-table-pane.tsx b/web-console/src/components/record-table-pane/record-table-pane.tsx
index b8629eb736..1fe729d64d 100644
--- a/web-console/src/components/record-table-pane/record-table-pane.tsx
+++ b/web-console/src/components/record-table-pane/record-table-pane.tsx
@@ -30,6 +30,7 @@ import { SMALL_TABLE_PAGE_SIZE, SMALL_TABLE_PAGE_SIZE_OPTIONS } from '../../reac
import type { Pagination } from '../../utils';
import {
columnToIcon,
+ columnToSummary,
columnToWidth,
filterMap,
formatNumber,
@@ -140,7 +141,7 @@ export const RecordTablePane = React.memo(function RecordTablePane(props: Record
Header() {
return (
<div className="clickable-cell">
- <div className="output-name">
+ <div className="output-name" title={columnToSummary(column)}>
{icon && <Icon className="type-icon" icon={icon} size={12} />}
{h}
{hasFilterOnHeader(h, i) && <Icon icon={IconNames.FILTER} size={14} />}
diff --git a/web-console/src/druid-models/execution/execution.ts b/web-console/src/druid-models/execution/execution.ts
index 9388d49283..97ecc14b2f 100644
--- a/web-console/src/druid-models/execution/execution.ts
+++ b/web-console/src/druid-models/execution/execution.ts
@@ -67,7 +67,7 @@ type ExecutionDestination =
| {
type: 'taskReport';
}
- | { type: 'dataSource'; dataSource: string; exists?: boolean }
+ | { type: 'dataSource'; dataSource: string; loaded?: boolean }
| { type: 'download' };
export type ExecutionStatus = 'RUNNING' | 'FAILED' | 'SUCCESS';
@@ -505,7 +505,7 @@ export class Execution {
});
}
- public markDestinationDatasourceExists(): Execution {
+ public markDestinationDatasourceLoaded(): Execution {
const { destination } = this;
if (destination?.type !== 'dataSource') return this;
@@ -513,7 +513,7 @@ export class Execution {
...this.valueOf(),
destination: {
...destination,
- exists: true,
+ loaded: true,
},
});
}
@@ -537,7 +537,7 @@ export class Execution {
const { status, destination } = this;
if (status === 'SUCCESS' && destination?.type === 'dataSource') {
- return Boolean(destination.exists);
+ return Boolean(destination.loaded);
}
return true;
diff --git a/web-console/src/helpers/execution/general.ts b/web-console/src/helpers/execution/general.ts
index bbb2eead6f..9c6e04fd1d 100644
--- a/web-console/src/helpers/execution/general.ts
+++ b/web-console/src/helpers/execution/general.ts
@@ -23,7 +23,7 @@ import type { Execution } from '../../druid-models';
import { IntermediateQueryState } from '../../utils';
import {
- updateExecutionWithDatasourceExistsIfNeeded,
+ updateExecutionWithDatasourceLoadedIfNeeded,
updateExecutionWithTaskIfNeeded,
} from './sql-task-execution';
@@ -49,7 +49,7 @@ export async function executionBackgroundStatusCheck(
switch (execution.engine) {
case 'sql-msq-task':
execution = await updateExecutionWithTaskIfNeeded(execution, cancelToken);
- execution = await updateExecutionWithDatasourceExistsIfNeeded(execution, cancelToken);
+ execution = await updateExecutionWithDatasourceLoadedIfNeeded(execution, cancelToken);
break;
default:
diff --git a/web-console/src/helpers/execution/sql-task-execution.ts b/web-console/src/helpers/execution/sql-task-execution.ts
index af5e40f315..c8afc1bf70 100644
--- a/web-console/src/helpers/execution/sql-task-execution.ts
+++ b/web-console/src/helpers/execution/sql-task-execution.ts
@@ -17,7 +17,7 @@
*/
import type { AxiosResponse, CancelToken } from 'axios';
-import { L } from 'druid-query-toolkit';
+import { L, QueryResult } from 'druid-query-toolkit';
import type { QueryContext } from '../../druid-models';
import { Execution } from '../../druid-models';
@@ -31,7 +31,8 @@ import {
} from '../../utils';
import { maybeGetClusterCapacity } from '../capacity';
-const WAIT_FOR_SEGMENTS_TIMEOUT = 180000; // 3 minutes to wait until segments appear
+const WAIT_FOR_SEGMENT_METADATA_TIMEOUT = 180000; // 3 minutes to wait until segments appear in the metadata
+const WAIT_FOR_SEGMENT_LOAD_TIMEOUT = 540000; // 9 minutes to wait for segments to load at all
export interface SubmitTaskQueryOptions {
query: string | Record<string, any>;
@@ -94,7 +95,17 @@ export async function submitTaskQuery(
throw new DruidError(druidError, prefixLines);
}
- let execution = Execution.fromTaskSubmit(sqlTaskResp.data, sqlQuery, context);
+ const sqlTaskPayload = sqlTaskResp.data;
+
+ if (!sqlTaskPayload.taskId) {
+ if (!Array.isArray(sqlTaskPayload)) throw new Error('unexpected task payload');
+ return Execution.fromResult(
+ 'sql-msq-task',
+ QueryResult.fromRawResult(sqlTaskPayload, false, true, true, true),
+ );
+ }
+
+ let execution = Execution.fromTaskSubmit(sqlTaskPayload, sqlQuery, context);
if (onSubmitted) {
onSubmitted(execution.id);
@@ -104,7 +115,7 @@ export async function submitTaskQuery(
execution = execution.changeDestination({ type: 'download' });
}
- execution = await updateExecutionWithDatasourceExistsIfNeeded(execution, cancelToken);
+ execution = await updateExecutionWithDatasourceLoadedIfNeeded(execution, cancelToken);
if (execution.isFullyComplete()) return execution;
@@ -129,7 +140,7 @@ export async function reattachTaskExecution(
try {
execution = await getTaskExecution(id, undefined, cancelToken);
- execution = await updateExecutionWithDatasourceExistsIfNeeded(execution, cancelToken);
+ execution = await updateExecutionWithDatasourceLoadedIfNeeded(execution, cancelToken);
} catch (e) {
throw new Error(`Reattaching to query failed due to: ${e.message}`);
}
@@ -217,17 +228,31 @@ export async function getTaskExecution(
return Execution.fromTaskStatus(statusResp.data);
}
-export async function updateExecutionWithDatasourceExistsIfNeeded(
+export async function updateExecutionWithDatasourceLoadedIfNeeded(
execution: Execution,
_cancelToken?: CancelToken,
): Promise<Execution> {
if (
- !(execution.destination?.type === 'dataSource' && !execution.destination.exists) ||
+ !(execution.destination?.type === 'dataSource' && !execution.destination.loaded) ||
execution.status !== 'SUCCESS'
) {
return execution;
}
+ const endTime = execution.getEndTime();
+ if (
+ !endTime || // If endTime is not set (this is not expected to happen) then just bow out
+ execution.stages?.getLastStage()?.partitionCount === 0 || // No data was meant to be written anyway, nothing to do
+ endTime.valueOf() + WAIT_FOR_SEGMENT_LOAD_TIMEOUT < Date.now() // Enough time has passed since the query ran... don't bother waiting for segments to load.
+ ) {
+ return execution.markDestinationDatasourceLoaded();
+ }
+
+ // Ideally we would have a more accurate query here, instead of
+ // COUNT(*) FILTER (WHERE is_published = 1 AND is_available = 0)
+ // we want to filter on something like
+ // COUNT(*) FILTER (WHERE is_should_be_available = 1 AND is_available = 0)
+ // `is_published` does not quite capture what we want but this is the best we have for now.
const segmentCheck = await queryDruidSql({
query: `SELECT
COUNT(*) AS num_segments,
@@ -239,21 +264,11 @@ WHERE datasource = ${L(execution.destination.dataSource)} AND is_overshadowed =
const numSegments: number = deepGet(segmentCheck, '0.num_segments') || 0;
const loadingSegments: number = deepGet(segmentCheck, '0.loading_segments') || 0;
- // There appear to be no segments either nothing was written out or they have not shown up in the metadata yet
+ // There appear to be no segments, since we checked above that something was written out we know that they have not shown up in the metadata yet
if (numSegments === 0) {
- const { stages } = execution;
- if (stages) {
- const lastStage = stages.getStage(stages.stageCount() - 1);
- if (lastStage.partitionCount === 0) {
- // No data was meant to be written anyway
- return execution.markDestinationDatasourceExists();
- }
- }
-
- const endTime = execution.getEndTime();
- if (!endTime || endTime.valueOf() + WAIT_FOR_SEGMENTS_TIMEOUT < Date.now()) {
- // Enough time has passed since the query ran... give up waiting (or there is no time info).
- return execution.markDestinationDatasourceExists();
+ if (endTime.valueOf() + WAIT_FOR_SEGMENT_METADATA_TIMEOUT < Date.now()) {
+ // Enough time has passed since the query ran... give up waiting for segments to show up in metadata.
+ return execution.markDestinationDatasourceLoaded();
}
return execution;
@@ -262,7 +277,7 @@ WHERE datasource = ${L(execution.destination.dataSource)} AND is_overshadowed =
// There are segments, and we are still waiting for some of them to load
if (loadingSegments > 0) return execution;
- return execution.markDestinationDatasourceExists();
+ return execution.markDestinationDatasourceLoaded();
}
function cancelTaskExecutionOnCancel(
diff --git a/web-console/src/utils/types.ts b/web-console/src/utils/types.ts
index ffb43d0115..35324c1fb5 100644
--- a/web-console/src/utils/types.ts
+++ b/web-console/src/utils/types.ts
@@ -20,6 +20,13 @@ import type { IconName } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import type { Column } from 'druid-query-toolkit';
+export function columnToSummary(column: Column): string {
+ const lines: string[] = [column.name];
+ if (column.sqlType) lines.push(`SQL type: ${column.sqlType}`);
+ if (column.nativeType) lines.push(`Native type: ${column.nativeType}`);
+ return lines.join('\n');
+}
+
function getEffectiveColumnType(column: Column): string | undefined {
if (column.sqlType === 'TIMESTAMP') return column.sqlType;
return column.nativeType || column.sqlType;
@@ -42,24 +49,35 @@ export function dataTypeToIcon(dataType: string): IconName {
return IconNames.FONT;
case 'BIGINT':
+ case 'LONG':
+ return IconNames.NUMERICAL;
+
case 'DECIMAL':
case 'REAL':
- case 'LONG':
case 'FLOAT':
case 'DOUBLE':
- return IconNames.NUMERICAL;
+ return IconNames.FLOATING_POINT;
case 'ARRAY<STRING>':
return IconNames.ARRAY_STRING;
case 'ARRAY<LONG>':
+ return IconNames.ARRAY_NUMERIC;
+
case 'ARRAY<FLOAT>':
case 'ARRAY<DOUBLE>':
- return IconNames.ARRAY_NUMERIC;
+ return IconNames.ARRAY_FLOATING_POINT;
case 'COMPLEX<JSON>':
return IconNames.DIAGRAM_TREE;
+ case 'COMPLEX<VARIANCE>':
+ return IconNames.ALIGNMENT_HORIZONTAL_CENTER;
+
+ case 'COMPLEX<IPADDRESS>':
+ case 'COMPLEX<IPPREFIX>':
+ return IconNames.IP_ADDRESS;
+
case 'NULL':
return IconNames.CIRCLE;
diff --git a/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.tsx b/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.tsx
index 810bc3bd74..c379e9322b 100644
--- a/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.tsx
+++ b/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.tsx
@@ -30,7 +30,13 @@ import { BracedText, Deferred, TableCell } from '../../../../components';
import { CellFilterMenu } from '../../../../components/cell-filter-menu/cell-filter-menu';
import { ShowValueDialog } from '../../../../dialogs/show-value-dialog/show-value-dialog';
import type { QueryAction } from '../../../../utils';
-import { columnToIcon, columnToWidth, filterMap, getNumericColumnBraces } from '../../../../utils';
+import {
+ columnToIcon,
+ columnToSummary,
+ columnToWidth,
+ filterMap,
+ getNumericColumnBraces,
+} from '../../../../utils';
import './preview-table.scss';
@@ -124,7 +130,7 @@ export const PreviewTable = React.memo(function PreviewTable(props: PreviewTable
Header() {
return (
<div className="header-wrapper" onClick={() => onEditColumn(i)}>
- <div className="output-name">
+ <div className="output-name" title={columnToSummary(column)}>
{icon && <Icon className="type-icon" icon={icon} size={12} />}
{h}
{hasFilterOnHeader(h, i) && (
diff --git a/web-console/src/views/workbench-view/column-tree/__snapshots__/column-tree.spec.tsx.snap b/web-console/src/views/workbench-view/column-tree/__snapshots__/column-tree.spec.tsx.snap
index f09caaf102..f1ed721698 100644
--- a/web-console/src/views/workbench-view/column-tree/__snapshots__/column-tree.spec.tsx.snap
+++ b/web-console/src/views/workbench-view/column-tree/__snapshots__/column-tree.spec.tsx.snap
@@ -98,7 +98,7 @@ exports[`ColumnTree matches snapshot 1`] = `
</Blueprint4.Popover2>,
},
Object {
- "icon": "numerical",
+ "icon": "floating-point",
"id": "addedBy10",
"label": <Blueprint4.Popover2
autoFocus={false}
diff --git a/web-console/src/views/workbench-view/execution-progress-bar-pane/__snapshots__/execution-progress-bar-pane.spec.tsx.snap b/web-console/src/views/workbench-view/execution-progress-bar-pane/__snapshots__/execution-progress-bar-pane.spec.tsx.snap
index f2e9970e58..68a8e3103c 100644
--- a/web-console/src/views/workbench-view/execution-progress-bar-pane/__snapshots__/execution-progress-bar-pane.spec.tsx.snap
+++ b/web-console/src/views/workbench-view/execution-progress-bar-pane/__snapshots__/execution-progress-bar-pane.spec.tsx.snap
@@ -12,7 +12,7 @@ exports[`ExecutionProgressBarPane matches snapshot 1`] = `
className="cancel"
onClick={[Function]}
>
- (stop waiting)
+ (skip waiting)
</span>
</React.Fragment>
</Unknown>
diff --git a/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx b/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx
index 24a669d4a5..0e146327d8 100644
--- a/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx
+++ b/web-console/src/views/workbench-view/execution-progress-bar-pane/execution-progress-bar-pane.tsx
@@ -57,7 +57,7 @@ export const ExecutionProgressBarPane = React.memo(function ExecutionProgressBar
<>
{' '}
<span className="cancel" onClick={cancelMaybeConfirm}>
- {stages && !execution.isWaitingForQuery() ? '(stop waiting)' : '(cancel)'}
+ {stages && !execution.isWaitingForQuery() ? '(skip waiting)' : '(cancel)'}
</span>
</>
)}
diff --git a/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx b/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx
index 743e44174f..8571ae5996 100644
--- a/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx
+++ b/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx
@@ -39,6 +39,7 @@ import { SMALL_TABLE_PAGE_SIZE, SMALL_TABLE_PAGE_SIZE_OPTIONS } from '../../../r
import type { Pagination, QueryAction } from '../../../utils';
import {
columnToIcon,
+ columnToSummary,
columnToWidth,
convertToGroupByExpression,
copyAndAlert,
@@ -587,7 +588,7 @@ export const ResultTablePane = React.memo(function ResultTablePane(props: Result
return (
<Popover2 content={<Deferred content={() => getHeaderMenu(column, i)} />}>
<div className="clickable-cell">
- <div className="output-name">
+ <div className="output-name" title={columnToSummary(column)}>
{icon && <Icon className="type-icon" icon={icon} size={12} />}
{h}
{hasFilterOnHeader(h, i) && (
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org