You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by vo...@apache.org on 2024/03/18 19:58:42 UTC
(druid) branch 29.0.1 updated: Web console: Make array ingest mode ux better (#15927) (#16151)
This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch 29.0.1
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/29.0.1 by this push:
new 585ea963da3 Web console: Make array ingest mode ux better (#15927) (#16151)
585ea963da3 is described below
commit 585ea963da382e645e4ad4d71b0dbf5839dc477b
Author: Karan Kumar <ka...@gmail.com>
AuthorDate: Tue Mar 19 01:28:36 2024 +0530
Web console: Make array ingest mode ux better (#15927) (#16151)
* only set arrayIngestMode: array when needed (still do the queries the correct way)
* arrayIngestMode control
* update wording
* feedback fixes
Co-authored-by: Vadim Ogievetsky <va...@ogievetsky.com>
---
.../druid-models/ingestion-spec/ingestion-spec.tsx | 7 ++-
.../druid-models/query-context/query-context.tsx | 21 +++++++-
.../workbench-query/workbench-query.ts | 6 +--
web-console/src/helpers/spec-conversion.spec.ts | 19 +++-----
web-console/src/helpers/spec-conversion.ts | 19 +++++---
.../sql-data-loader-view/sql-data-loader-view.tsx | 5 +-
.../views/workbench-view/run-panel/run-panel.tsx | 56 +++++++++++++++++++++-
.../src/views/workbench-view/workbench-view.tsx | 9 +---
8 files changed, 107 insertions(+), 35 deletions(-)
diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
index bd9a8989225..838c93c4f82 100644
--- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
+++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
@@ -297,7 +297,10 @@ export function getSchemaMode(spec: Partial<IngestionSpec>): SchemaMode {
return Array.isArray(dimensions) && dimensions.length === 0 ? 'string-only-discovery' : 'fixed';
}
-export function getArrayMode(spec: Partial<IngestionSpec>): ArrayMode {
+export function getArrayMode(
+ spec: Partial<IngestionSpec>,
+ whenUnclear: ArrayMode = 'arrays',
+): ArrayMode {
const schemaMode = getSchemaMode(spec);
switch (schemaMode) {
case 'type-aware-discovery':
@@ -332,7 +335,7 @@ export function getArrayMode(spec: Partial<IngestionSpec>): ArrayMode {
return 'multi-values';
}
- return 'arrays';
+ return whenUnclear;
}
}
}
diff --git a/web-console/src/druid-models/query-context/query-context.tsx b/web-console/src/druid-models/query-context/query-context.tsx
index 838ab1f3a50..17e8204e949 100644
--- a/web-console/src/druid-models/query-context/query-context.tsx
+++ b/web-console/src/druid-models/query-context/query-context.tsx
@@ -18,6 +18,8 @@
import { deepDelete, deepSet } from '../../utils';
+export type ArrayIngestMode = 'array' | 'mvd';
+
export interface QueryContext {
useCache?: boolean;
populateCache?: boolean;
@@ -32,7 +34,7 @@ export interface QueryContext {
durableShuffleStorage?: boolean;
maxParseExceptions?: number;
groupByEnableMultiValueUnnesting?: boolean;
- arrayIngestMode?: 'array' | 'mvd';
+ arrayIngestMode?: ArrayIngestMode;
[key: string]: any;
}
@@ -248,3 +250,20 @@ export function changeMaxParseExceptions(
return deepDelete(context, 'maxParseExceptions');
}
}
+
+// arrayIngestMode
+
+export function getArrayIngestMode(context: QueryContext): ArrayIngestMode | undefined {
+ return context.arrayIngestMode;
+}
+
+export function changeArrayIngestMode(
+ context: QueryContext,
+ arrayIngestMode: ArrayIngestMode | undefined,
+): QueryContext {
+ if (arrayIngestMode) {
+ return deepSet(context, 'arrayIngestMode', arrayIngestMode);
+ } else {
+ return deepDelete(context, 'arrayIngestMode');
+ }
+}
diff --git a/web-console/src/druid-models/workbench-query/workbench-query.ts b/web-console/src/druid-models/workbench-query/workbench-query.ts
index 27cf6a0109d..d59cdbbe92e 100644
--- a/web-console/src/druid-models/workbench-query/workbench-query.ts
+++ b/web-console/src/druid-models/workbench-query/workbench-query.ts
@@ -94,6 +94,8 @@ export class WorkbenchQuery {
partitionedByHint: string | undefined,
arrayMode: ArrayMode,
): WorkbenchQuery {
+ const queryContext: QueryContext = {};
+ if (arrayMode === 'arrays') queryContext.arrayIngestMode = 'array';
return new WorkbenchQuery({
queryString: ingestQueryPatternToQuery(
externalConfigToIngestQueryPattern(
@@ -103,9 +105,7 @@ export class WorkbenchQuery {
arrayMode,
),
).toString(),
- queryContext: {
- arrayIngestMode: 'array',
- },
+ queryContext,
});
}
diff --git a/web-console/src/helpers/spec-conversion.spec.ts b/web-console/src/helpers/spec-conversion.spec.ts
index bea3f730318..9271196dafa 100644
--- a/web-console/src/helpers/spec-conversion.spec.ts
+++ b/web-console/src/helpers/spec-conversion.spec.ts
@@ -123,10 +123,7 @@ describe('spec conversion', () => {
expect(converted.queryString).toMatchSnapshot();
expect(converted.queryContext).toEqual({
- arrayIngestMode: 'array',
- groupByEnableMultiValueUnnesting: false,
maxParseExceptions: 3,
- finalizeAggregations: false,
maxNumTasks: 5,
indexSpec: {
dimensionCompression: 'lzf',
@@ -232,11 +229,7 @@ describe('spec conversion', () => {
expect(converted.queryString).toMatchSnapshot();
- expect(converted.queryContext).toEqual({
- arrayIngestMode: 'array',
- groupByEnableMultiValueUnnesting: false,
- finalizeAggregations: false,
- });
+ expect(converted.queryContext).toEqual({});
});
it('converts index_hadoop spec (with rollup)', () => {
@@ -357,11 +350,7 @@ describe('spec conversion', () => {
expect(converted.queryString).toMatchSnapshot();
- expect(converted.queryContext).toEqual({
- arrayIngestMode: 'array',
- groupByEnableMultiValueUnnesting: false,
- finalizeAggregations: false,
- });
+ expect(converted.queryContext).toEqual({});
});
it('converts with issue when there is a __time transform', () => {
@@ -663,5 +652,9 @@ describe('spec conversion', () => {
});
expect(converted.queryString).toMatchSnapshot();
+
+ expect(converted.queryContext).toEqual({
+ arrayIngestMode: 'array',
+ });
});
});
diff --git a/web-console/src/helpers/spec-conversion.ts b/web-console/src/helpers/spec-conversion.ts
index f25406e5028..7e7673c7384 100644
--- a/web-console/src/helpers/spec-conversion.ts
+++ b/web-console/src/helpers/spec-conversion.ts
@@ -32,11 +32,18 @@ import type {
DimensionSpec,
IngestionSpec,
MetricSpec,
+ QueryContext,
QueryWithContext,
TimestampSpec,
Transform,
} from '../druid-models';
-import { inflateDimensionSpec, NO_SUCH_COLUMN, TIME_COLUMN, upgradeSpec } from '../druid-models';
+import {
+ getArrayMode,
+ inflateDimensionSpec,
+ NO_SUCH_COLUMN,
+ TIME_COLUMN,
+ upgradeSpec,
+} from '../druid-models';
import { deepGet, filterMap, nonEmptyArray, oneOf } from '../utils';
export function getSpecDatasourceName(spec: Partial<IngestionSpec>): string {
@@ -73,11 +80,11 @@ export function convertSpecToSql(spec: any): QueryWithContext {
}
spec = upgradeSpec(spec, true);
- const context: Record<string, any> = {
- finalizeAggregations: false,
- groupByEnableMultiValueUnnesting: false,
- arrayIngestMode: 'array',
- };
+ const context: QueryContext = {};
+
+ if (getArrayMode(spec, 'multi-values') === 'arrays') {
+ context.arrayIngestMode = 'array';
+ }
const indexSpec = deepGet(spec, 'spec.tuningConfig.indexSpec');
if (indexSpec) {
diff --git a/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx b/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
index 0fc89573bd2..27017f725da 100644
--- a/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
+++ b/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
@@ -48,7 +48,6 @@ import './sql-data-loader-view.scss';
const INITIAL_QUERY_CONTEXT: QueryContext = {
finalizeAggregations: false,
groupByEnableMultiValueUnnesting: false,
- arrayIngestMode: 'array',
};
interface LoaderContent extends QueryWithContext {
@@ -190,6 +189,8 @@ export const SqlDataLoaderView = React.memo(function SqlDataLoaderView(
initInputFormat={inputFormat}
doneButton={false}
onSet={({ inputSource, inputFormat, signature, timeExpression, arrayMode }) => {
+ const queryContext: QueryContext = { ...INITIAL_QUERY_CONTEXT };
+ if (arrayMode === 'arrays') queryContext.arrayIngestMode = 'array';
setContent({
queryString: ingestQueryPatternToQuery(
externalConfigToIngestQueryPattern(
@@ -199,7 +200,7 @@ export const SqlDataLoaderView = React.memo(function SqlDataLoaderView(
arrayMode,
),
).toString(),
- queryContext: INITIAL_QUERY_CONTEXT,
+ queryContext,
});
}}
altText="Skip the wizard and continue with custom SQL"
diff --git a/web-console/src/views/workbench-view/run-panel/run-panel.tsx b/web-console/src/views/workbench-view/run-panel/run-panel.tsx
index ac67c40ff27..5d9c9852791 100644
--- a/web-console/src/views/workbench-view/run-panel/run-panel.tsx
+++ b/web-console/src/views/workbench-view/run-panel/run-panel.tsx
@@ -25,6 +25,7 @@ import {
MenuDivider,
MenuItem,
Position,
+ Tag,
useHotkeys,
} from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
@@ -35,8 +36,15 @@ import React, { useCallback, useMemo, useState } from 'react';
import { MenuCheckbox, MenuTristate } from '../../../components';
import { EditContextDialog, StringInputDialog } from '../../../dialogs';
import { IndexSpecDialog } from '../../../dialogs/index-spec-dialog/index-spec-dialog';
-import type { DruidEngine, IndexSpec, QueryContext, WorkbenchQuery } from '../../../druid-models';
+import type {
+ ArrayIngestMode,
+ DruidEngine,
+ IndexSpec,
+ QueryContext,
+ WorkbenchQuery,
+} from '../../../druid-models';
import {
+ changeArrayIngestMode,
changeDurableShuffleStorage,
changeFailOnEmptyInsert,
changeFinalizeAggregations,
@@ -47,6 +55,7 @@ import {
changeUseApproximateTopN,
changeUseCache,
changeWaitUntilSegmentsLoad,
+ getArrayIngestMode,
getDurableShuffleStorage,
getFailOnEmptyInsert,
getFinalizeAggregations,
@@ -59,6 +68,7 @@ import {
getWaitUntilSegmentsLoad,
summarizeIndexSpec,
} from '../../../druid-models';
+import { getLink } from '../../../links';
import { deepGet, deepSet, pluralIfNeeded, tickIcon } from '../../../utils';
import { MaxTasksButton } from '../max-tasks-button/max-tasks-button';
import { QueryParametersDialog } from '../query-parameters-dialog/query-parameters-dialog';
@@ -87,6 +97,20 @@ const NAMED_TIMEZONES: string[] = [
'Australia/Sydney', // +11.0
];
+const ARRAY_INGEST_MODE_DESCRIPTION: Record<ArrayIngestMode, JSX.Element> = {
+ array: (
+ <>
+ array: Load SQL <Tag minimal>VARCHAR ARRAY</Tag> as Druid{' '}
+ <Tag minimal>ARRAY<STRING></Tag>
+ </>
+ ),
+ mvd: (
+ <>
+ mvd: Load SQL <Tag minimal>VARCHAR ARRAY</Tag> as Druid multi-value <Tag minimal>STRING</Tag>
+ </>
+ ),
+};
+
export interface RunPanelProps {
query: WorkbenchQuery;
onQueryChange(query: WorkbenchQuery): void;
@@ -112,6 +136,7 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
const numContextKeys = Object.keys(queryContext).length;
const queryParameters = query.queryParameters;
+ const arrayIngestMode = getArrayIngestMode(queryContext);
const maxParseExceptions = getMaxParseExceptions(queryContext);
const failOnEmptyInsert = getFailOnEmptyInsert(queryContext);
const finalizeAggregations = getFinalizeAggregations(queryContext);
@@ -472,6 +497,35 @@ export const RunPanel = React.memo(function RunPanel(props: RunPanelProps) {
changeQueryContext={changeQueryContext}
/>
)}
+ {ingestMode && (
+ <Popover2
+ position={Position.BOTTOM_LEFT}
+ content={
+ <Menu>
+ {([undefined, 'array', 'mvd'] as (ArrayIngestMode | undefined)[]).map((m, i) => (
+ <MenuItem
+ key={i}
+ icon={tickIcon(m === arrayIngestMode)}
+ text={m ? ARRAY_INGEST_MODE_DESCRIPTION[m] : '(server default)'}
+ onClick={() => changeQueryContext(changeArrayIngestMode(queryContext, m))}
+ />
+ ))}
+ <MenuDivider />
+ <MenuItem
+ icon={IconNames.HELP}
+ text="Documentation"
+ href={`${getLink('DOCS')}/querying/arrays#arrayingestmode`}
+ target="_blank"
+ />
+ </Menu>
+ }
+ >
+ <Button
+ text={`Array ingest mode: ${arrayIngestMode ?? '(server default)'}`}
+ rightIcon={IconNames.CARET_DOWN}
+ />
+ </Popover2>
+ )}
</ButtonGroup>
)}
{moreMenu && (
diff --git a/web-console/src/views/workbench-view/workbench-view.tsx b/web-console/src/views/workbench-view/workbench-view.tsx
index 7d721da413f..76699f52a08 100644
--- a/web-console/src/views/workbench-view/workbench-view.tsx
+++ b/web-console/src/views/workbench-view/workbench-view.tsx
@@ -320,18 +320,13 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
return (
<ConnectExternalDataDialog
- onSetExternalConfig={(
- externalConfig,
- timeExpression,
- partitionedByHint,
- forceMultiValue,
- ) => {
+ onSetExternalConfig={(externalConfig, timeExpression, partitionedByHint, arrayMode) => {
this.handleNewTab(
WorkbenchQuery.fromInitExternalConfig(
externalConfig,
timeExpression,
partitionedByHint,
- forceMultiValue,
+ arrayMode,
),
'Ext ' + guessDataSourceNameFromInputSource(externalConfig.inputSource),
);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org