You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by su...@apache.org on 2021/08/05 19:30:49 UTC
[druid] branch master updated: Web console: fix count aggregation
input in the data loader (#11485)
This is an automated email from the ASF dual-hosted git repository.
suneet pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 257bc5c Web console: fix count aggregation input in the data loader (#11485)
257bc5c is described below
commit 257bc5c62fda5966e6c22b2d95be3fcf433be1c8
Author: Vadim Ogievetsky <va...@ogievetsky.com>
AuthorDate: Thu Aug 5 12:30:30 2021 -0700
Web console: fix count aggregation input in the data loader (#11485)
* add typeIs
* fix unused field in count metric
* better types
* typos
* work with readonly types
* factor out apply cancel buttons
* form editor
* selection type
* unsaved changes
* form editor spec
* tidy up sampler
* more menu controls
* update e2e test
---
web-console/e2e-tests/reindexing.spec.ts | 24 +-
web-console/src/components/auto-form/auto-form.tsx | 27 +-
.../lookup-edit-dialog/lookup-edit-dialog.tsx | 7 +-
web-console/src/druid-models/compaction-config.tsx | 25 +-
web-console/src/druid-models/dimension-spec.ts | 19 +-
web-console/src/druid-models/filter.tsx | 37 +-
web-console/src/druid-models/flatten-spec.tsx | 4 +-
web-console/src/druid-models/ingestion-spec.tsx | 144 ++---
web-console/src/druid-models/input-format.tsx | 46 +-
web-console/src/druid-models/lookup-spec.tsx | 195 +++---
web-console/src/druid-models/metric-spec.tsx | 94 +--
web-console/src/druid-models/timestamp-spec.tsx | 18 +-
web-console/src/druid-models/transform-spec.tsx | 10 +-
web-console/src/utils/general.tsx | 12 +-
web-console/src/utils/sampler.ts | 58 +-
.../load-data-view/filter-table/filter-table.tsx | 2 +-
.../__snapshots__/form-editor.spec.tsx.snap | 113 ++++
.../load-data-view/form-editor/form-editor.scss | 42 ++
.../form-editor/form-editor.spec.tsx | 48 ++
.../load-data-view/form-editor/form-editor.tsx | 94 +++
.../src/views/load-data-view/info-messages.tsx | 2 +-
.../src/views/load-data-view/load-data-view.scss | 10 +-
.../src/views/load-data-view/load-data-view.tsx | 686 +++++++++------------
.../parse-time-table/parse-time-table.tsx | 2 +-
.../load-data-view/schema-table/schema-table.tsx | 10 +-
.../transform-table/transform-table.tsx | 4 +-
.../src/views/lookups-view/lookups-view.tsx | 2 +-
.../views/query-view/column-tree/column-tree.tsx | 73 ++-
28 files changed, 1032 insertions(+), 776 deletions(-)
diff --git a/web-console/e2e-tests/reindexing.spec.ts b/web-console/e2e-tests/reindexing.spec.ts
index 3a2baf9..91952c8 100644
--- a/web-console/e2e-tests/reindexing.spec.ts
+++ b/web-console/e2e-tests/reindexing.spec.ts
@@ -128,15 +128,9 @@ function validateConnectLocalData(preview: string) {
',"namespace":"Talk"' +
',"page":"Talk:Oswald Tilghman"' +
',"user":"GELongstreet"' +
- ',"added":"36"' +
- ',"deleted":"0"' +
- ',"delta":"36"' +
- ',"cityName":null' +
- ',"countryIsoCode":null' +
- ',"countryName":null' +
- ',"regionIsoCode":null' +
- ',"regionName":null' +
- ',"metroCode":null' +
+ ',"added":36' +
+ ',"deleted":0' +
+ ',"delta":36' +
'}',
);
const lastLine = lines[lines.length - 1];
@@ -153,15 +147,9 @@ function validateConnectLocalData(preview: string) {
',"namespace":"Main"' +
',"page":"Hapoel Katamon Jerusalem F.C."' +
',"user":"The Quixotic Potato"' +
- ',"added":"1"' +
- ',"deleted":"0"' +
- ',"delta":"1"' +
- ',"cityName":null' +
- ',"countryIsoCode":null' +
- ',"countryName":null' +
- ',"regionIsoCode":null' +
- ',"regionName":null' +
- ',"metroCode":null' +
+ ',"added":1' +
+ ',"deleted":0' +
+ ',"delta":1' +
'}',
);
}
diff --git a/web-console/src/components/auto-form/auto-form.tsx b/web-console/src/components/auto-form/auto-form.tsx
index ee67720..f2f13a2 100644
--- a/web-console/src/components/auto-form/auto-form.tsx
+++ b/web-console/src/components/auto-form/auto-form.tsx
@@ -31,7 +31,7 @@ import { SuggestibleInput, Suggestion } from '../suggestible-input/suggestible-i
import './auto-form.scss';
-export type Functor<M, R> = R | ((model: M) => R);
+export type Functor<M, R> = R | ((model: Partial<M>) => R);
export interface Field<M> {
name: string;
@@ -59,7 +59,7 @@ export interface Field<M> {
hide?: Functor<M, boolean>;
hideInMore?: Functor<M, boolean>;
valueAdjustment?: (value: any) => any;
- adjustment?: (model: M) => M;
+ adjustment?: (model: Partial<M>) => Partial<M>;
issueWithValue?: (value: any) => string | undefined;
}
@@ -71,12 +71,12 @@ interface ComputedFieldValues {
export interface AutoFormProps<M> {
fields: Field<M>[];
- model: M | undefined;
- onChange: (newModel: M) => void;
+ model: Partial<M> | undefined;
+ onChange: (newModel: Partial<M>) => void;
onFinalize?: () => void;
- showCustom?: (model: M) => boolean;
+ showCustom?: (model: Partial<M>) => boolean;
large?: boolean;
- globalAdjustment?: (model: M) => M;
+ globalAdjustment?: (model: Partial<M>) => Partial<M>;
}
export interface AutoFormState {
@@ -111,7 +111,7 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
static evaluateFunctor<M, R>(
functor: undefined | Functor<M, R>,
- model: M | undefined,
+ model: Partial<M> | undefined,
defaultValue: R,
): R {
if (!model || functor == null) return defaultValue;
@@ -122,7 +122,14 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
}
}
- static issueWithModel<M>(model: M | undefined, fields: readonly Field<M>[]): string | undefined {
+ static isValidModel<M>(model: Partial<M> | undefined, fields: readonly Field<M>[]): model is M {
+ return !AutoForm.issueWithModel(model, fields);
+ }
+
+ static issueWithModel<M>(
+ model: Partial<M> | undefined,
+ fields: readonly Field<M>[],
+ ): string | undefined {
if (typeof model === 'undefined') {
return `model is undefined`;
}
@@ -179,7 +186,7 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
newValue = field.valueAdjustment(newValue);
}
- let newModel: T;
+ let newModel: Partial<T>;
if (typeof newValue === 'undefined') {
if (typeof field.emptyValue === 'undefined') {
newModel = deepDelete(model, field.name);
@@ -197,7 +204,7 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
this.modelChange(newModel);
};
- private readonly modelChange = (newModel: T) => {
+ private readonly modelChange = (newModel: Partial<T>) => {
const { globalAdjustment, fields, onChange, model } = this.props;
// Delete things that are not defined now (but were defined prior to the change)
diff --git a/web-console/src/dialogs/lookup-edit-dialog/lookup-edit-dialog.tsx b/web-console/src/dialogs/lookup-edit-dialog/lookup-edit-dialog.tsx
index 889f587..ffd4e49 100644
--- a/web-console/src/dialogs/lookup-edit-dialog/lookup-edit-dialog.tsx
+++ b/web-console/src/dialogs/lookup-edit-dialog/lookup-edit-dialog.tsx
@@ -35,11 +35,14 @@ import './lookup-edit-dialog.scss';
export interface LookupEditDialogProps {
onClose: () => void;
onSubmit: (updateLookupVersion: boolean) => void;
- onChange: (field: 'name' | 'tier' | 'version' | 'spec', value: string | LookupSpec) => void;
+ onChange: (
+ field: 'name' | 'tier' | 'version' | 'spec',
+ value: string | Partial<LookupSpec>,
+ ) => void;
lookupName: string;
lookupTier: string;
lookupVersion: string;
- lookupSpec: LookupSpec;
+ lookupSpec: Partial<LookupSpec>;
isEdit: boolean;
allLookupTiers: string[];
}
diff --git a/web-console/src/druid-models/compaction-config.tsx b/web-console/src/druid-models/compaction-config.tsx
index f7286e4..e29dffe 100644
--- a/web-console/src/druid-models/compaction-config.tsx
+++ b/web-console/src/druid-models/compaction-config.tsx
@@ -55,14 +55,14 @@ export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
name: 'tuningConfig.partitionsSpec.maxRowsPerSegment',
type: 'number',
defaultValue: 5000000,
- defined: (t: CompactionConfig) => deepGet(t, 'tuningConfig.partitionsSpec.type') === 'dynamic',
+ defined: t => deepGet(t, 'tuningConfig.partitionsSpec.type') === 'dynamic',
info: <>Determines how many rows are in each segment.</>,
},
{
name: 'tuningConfig.partitionsSpec.maxTotalRows',
type: 'number',
defaultValue: 20000000,
- defined: (t: CompactionConfig) => deepGet(t, 'tuningConfig.partitionsSpec.type') === 'dynamic',
+ defined: t => deepGet(t, 'tuningConfig.partitionsSpec.type') === 'dynamic',
info: <>Total number of rows in segments waiting for being pushed.</>,
},
// partitionsSpec type: hashed
@@ -71,7 +71,7 @@ export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
type: 'number',
zeroMeansUndefined: true,
placeholder: `(defaults to 500000)`,
- defined: (t: CompactionConfig) =>
+ defined: t =>
deepGet(t, 'tuningConfig.partitionsSpec.type') === 'hashed' &&
!deepGet(t, 'tuningConfig.partitionsSpec.numShards') &&
!deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'),
@@ -92,7 +92,7 @@ export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
name: 'tuningConfig.partitionsSpec.maxRowsPerSegment',
type: 'number',
zeroMeansUndefined: true,
- defined: (t: CompactionConfig) =>
+ defined: t =>
deepGet(t, 'tuningConfig.partitionsSpec.type') === 'hashed' &&
!deepGet(t, 'tuningConfig.partitionsSpec.numShards') &&
!deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment'),
@@ -113,7 +113,7 @@ export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
name: 'tuningConfig.partitionsSpec.numShards',
type: 'number',
zeroMeansUndefined: true,
- defined: (t: CompactionConfig) =>
+ defined: t =>
deepGet(t, 'tuningConfig.partitionsSpec.type') === 'hashed' &&
!deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment') &&
!deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment'),
@@ -135,15 +135,14 @@ export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
name: 'tuningConfig.partitionsSpec.partitionDimensions',
type: 'string-array',
placeholder: '(all dimensions)',
- defined: (t: CompactionConfig) => deepGet(t, 'tuningConfig.partitionsSpec.type') === 'hashed',
+ defined: t => deepGet(t, 'tuningConfig.partitionsSpec.type') === 'hashed',
info: <p>The dimensions to partition on. Leave blank to select all dimensions.</p>,
},
// partitionsSpec type: single_dim
{
name: 'tuningConfig.partitionsSpec.partitionDimension',
type: 'string',
- defined: (t: CompactionConfig) =>
- deepGet(t, 'tuningConfig.partitionsSpec.type') === 'single_dim',
+ defined: t => deepGet(t, 'tuningConfig.partitionsSpec.type') === 'single_dim',
required: true,
info: <p>The dimension to partition on.</p>,
},
@@ -151,7 +150,7 @@ export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
name: 'tuningConfig.partitionsSpec.targetRowsPerSegment',
type: 'number',
zeroMeansUndefined: true,
- defined: (t: CompactionConfig) =>
+ defined: t =>
deepGet(t, 'tuningConfig.partitionsSpec.type') === 'single_dim' &&
!deepGet(t, 'tuningConfig.partitionsSpec.maxRowsPerSegment'),
required: (t: CompactionConfig) =>
@@ -168,7 +167,7 @@ export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
name: 'tuningConfig.partitionsSpec.maxRowsPerSegment',
type: 'number',
zeroMeansUndefined: true,
- defined: (t: CompactionConfig) =>
+ defined: t =>
deepGet(t, 'tuningConfig.partitionsSpec.type') === 'single_dim' &&
!deepGet(t, 'tuningConfig.partitionsSpec.targetRowsPerSegment'),
required: (t: CompactionConfig) =>
@@ -180,8 +179,7 @@ export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
name: 'tuningConfig.partitionsSpec.assumeGrouped',
type: 'boolean',
defaultValue: false,
- defined: (t: CompactionConfig) =>
- deepGet(t, 'tuningConfig.partitionsSpec.type') === 'single_dim',
+ defined: t => deepGet(t, 'tuningConfig.partitionsSpec.type') === 'single_dim',
info: (
<p>
Assume that input data has already been grouped on time and dimensions. Ingestion will run
@@ -223,8 +221,7 @@ export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
type: 'number',
defaultValue: 10,
min: 1,
- defined: (t: CompactionConfig) =>
- oneOf(deepGet(t, 'tuningConfig.partitionsSpec.type'), 'hashed', 'single_dim'),
+ defined: t => oneOf(deepGet(t, 'tuningConfig.partitionsSpec.type'), 'hashed', 'single_dim'),
info: <>Maximum number of merge tasks which can be run at the same time.</>,
},
{
diff --git a/web-console/src/druid-models/dimension-spec.ts b/web-console/src/druid-models/dimension-spec.ts
index e305d67..4805b5b 100644
--- a/web-console/src/druid-models/dimension-spec.ts
+++ b/web-console/src/druid-models/dimension-spec.ts
@@ -17,38 +17,41 @@
*/
import { Field } from '../components';
-import { filterMap } from '../utils';
+import { filterMap, typeIs } from '../utils';
import { HeaderAndRows } from '../utils/sampler';
import { getColumnTypeFromHeaderAndRows } from './ingestion-spec';
export interface DimensionsSpec {
- dimensions?: (string | DimensionSpec)[];
- dimensionExclusions?: string[];
- spatialDimensions?: any[];
+ readonly dimensions?: (string | DimensionSpec)[];
+ readonly dimensionExclusions?: string[];
+ readonly spatialDimensions?: any[];
}
export interface DimensionSpec {
- type: string;
- name: string;
- createBitmapIndex?: boolean;
+ readonly type: string;
+ readonly name: string;
+ readonly createBitmapIndex?: boolean;
}
export const DIMENSION_SPEC_FIELDS: Field<DimensionSpec>[] = [
{
name: 'name',
type: 'string',
+ required: true,
+ placeholder: 'dimension_name',
},
{
name: 'type',
type: 'string',
+ required: true,
suggestions: ['string', 'long', 'float', 'double'],
},
{
name: 'createBitmapIndex',
type: 'boolean',
+ defined: typeIs('string'),
defaultValue: true,
- defined: (dimensionSpec: DimensionSpec) => dimensionSpec.type === 'string',
},
];
diff --git a/web-console/src/druid-models/filter.tsx b/web-console/src/druid-models/filter.tsx
index 5c11784..89cb2b6 100644
--- a/web-console/src/druid-models/filter.tsx
+++ b/web-console/src/druid-models/filter.tsx
@@ -20,11 +20,14 @@ import React from 'react';
import { ExternalLink, Field } from '../components';
import { getLink } from '../links';
-import { deepGet, EMPTY_ARRAY, oneOf } from '../utils';
+import { deepGet, EMPTY_ARRAY, oneOf, typeIs } from '../utils';
import { IngestionSpec } from './ingestion-spec';
-export type DruidFilter = Record<string, any>;
+export interface DruidFilter {
+ readonly type: string;
+ readonly [k: string]: any;
+}
export interface DimensionFiltersWithRest {
dimensionFilters: DruidFilter[];
@@ -81,33 +84,39 @@ export const FILTER_FIELDS: Field<DruidFilter>[] = [
{
name: 'type',
type: 'string',
+ required: true,
suggestions: KNOWN_FILTER_TYPES,
},
{
name: 'dimension',
type: 'string',
- defined: (df: DruidFilter) => oneOf(df.type, 'selector', 'in', 'interval', 'regex', 'like'),
+ defined: typeIs('selector', 'in', 'interval', 'regex', 'like'),
+ required: true,
},
{
name: 'value',
type: 'string',
- defined: (df: DruidFilter) => df.type === 'selector',
+ defined: typeIs('selector'),
+ required: true,
},
{
name: 'values',
type: 'string-array',
- defined: (df: DruidFilter) => df.type === 'in',
+ defined: typeIs('in'),
+ required: true,
},
{
name: 'intervals',
type: 'string-array',
- defined: (df: DruidFilter) => df.type === 'interval',
+ defined: typeIs('interval'),
+ required: true,
placeholder: 'ex: 2020-01-01/2020-06-01',
},
{
name: 'pattern',
type: 'string',
- defined: (df: DruidFilter) => oneOf(df.type, 'regex', 'like'),
+ defined: typeIs('regex', 'like'),
+ required: true,
},
{
@@ -115,39 +124,39 @@ export const FILTER_FIELDS: Field<DruidFilter>[] = [
label: 'Sub-filter type',
type: 'string',
suggestions: ['selector', 'in', 'interval', 'regex', 'like'],
- defined: (df: DruidFilter) => df.type === 'not',
+ defined: typeIs('not'),
+ required: true,
},
{
name: 'field.dimension',
label: 'Sub-filter dimension',
type: 'string',
- defined: (df: DruidFilter) => df.type === 'not',
+ defined: typeIs('not'),
},
{
name: 'field.value',
label: 'Sub-filter value',
type: 'string',
- defined: (df: DruidFilter) => df.type === 'not' && deepGet(df, 'field.type') === 'selector',
+ defined: df => df.type === 'not' && deepGet(df, 'field.type') === 'selector',
},
{
name: 'field.values',
label: 'Sub-filter values',
type: 'string-array',
- defined: (df: DruidFilter) => df.type === 'not' && deepGet(df, 'field.type') === 'in',
+ defined: df => df.type === 'not' && deepGet(df, 'field.type') === 'in',
},
{
name: 'field.intervals',
label: 'Sub-filter intervals',
type: 'string-array',
- defined: (df: DruidFilter) => df.type === 'not' && deepGet(df, 'field.type') === 'interval',
+ defined: df => df.type === 'not' && deepGet(df, 'field.type') === 'interval',
placeholder: 'ex: 2020-01-01/2020-06-01',
},
{
name: 'field.pattern',
label: 'Sub-filter pattern',
type: 'string',
- defined: (df: DruidFilter) =>
- df.type === 'not' && oneOf(deepGet(df, 'field.type'), 'regex', 'like'),
+ defined: df => df.type === 'not' && oneOf(deepGet(df, 'field.type'), 'regex', 'like'),
},
];
diff --git a/web-console/src/druid-models/flatten-spec.tsx b/web-console/src/druid-models/flatten-spec.tsx
index f4246f1..d88bcf5 100644
--- a/web-console/src/druid-models/flatten-spec.tsx
+++ b/web-console/src/druid-models/flatten-spec.tsx
@@ -20,7 +20,7 @@ import React from 'react';
import { ExternalLink, Field } from '../components';
import { getLink } from '../links';
-import { oneOf } from '../utils';
+import { typeIs } from '../utils';
export interface FlattenSpec {
useFieldDiscovery?: boolean;
@@ -50,7 +50,7 @@ export const FLATTEN_FIELD_FIELDS: Field<FlattenField>[] = [
name: 'expr',
type: 'string',
placeholder: '$.thing',
- defined: (flattenField: FlattenField) => oneOf(flattenField.type, 'path', 'jq'),
+ defined: typeIs('path', 'jq'),
required: true,
info: (
<>
diff --git a/web-console/src/druid-models/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec.tsx
index 4424ff2..20641f2 100644
--- a/web-console/src/druid-models/ingestion-spec.tsx
+++ b/web-console/src/druid-models/ingestion-spec.tsx
@@ -31,6 +31,7 @@ import {
EMPTY_OBJECT,
filterMap,
oneOf,
+ typeIs,
} from '../utils';
import { HeaderAndRows } from '../utils/sampler';
@@ -56,17 +57,17 @@ export const MAX_INLINE_DATA_LENGTH = 65536;
const CURRENT_YEAR = new Date().getUTCFullYear();
export interface IngestionSpec {
- type: IngestionType;
- spec: IngestionSpecInner;
+ readonly type: IngestionType;
+ readonly spec: IngestionSpecInner;
}
export interface IngestionSpecInner {
- ioConfig: IoConfig;
- dataSchema: DataSchema;
- tuningConfig?: TuningConfig;
+ readonly ioConfig: IoConfig;
+ readonly dataSchema: DataSchema;
+ readonly tuningConfig?: TuningConfig;
}
-export function isEmptyIngestionSpec(spec: IngestionSpec) {
+export function isEmptyIngestionSpec(spec: Partial<IngestionSpec>) {
return Object.keys(spec).length === 0;
}
@@ -105,7 +106,9 @@ function ingestionTypeToIoAndTuningConfigType(ingestionType: IngestionType): str
}
}
-export function getIngestionComboType(spec: IngestionSpec): IngestionComboType | undefined {
+export function getIngestionComboType(
+ spec: Partial<IngestionSpec>,
+): IngestionComboType | undefined {
const ioConfig = deepGet(spec, 'spec.ioConfig') || EMPTY_OBJECT;
switch (ioConfig.type) {
@@ -187,7 +190,7 @@ export function getIngestionImage(ingestionType: IngestionComboTypeWithExtra): s
return ingestionType;
}
-export function getIngestionDocLink(spec: IngestionSpec): string {
+export function getIngestionDocLink(spec: Partial<IngestionSpec>): string {
const type = getSpecType(spec);
switch (type) {
@@ -227,7 +230,7 @@ export function getRequiredModule(ingestionType: IngestionComboTypeWithExtra): s
}
}
-export function getIssueWithSpec(spec: IngestionSpec): string | undefined {
+export function getIssueWithSpec(spec: Partial<IngestionSpec>): string | undefined {
if (!deepGet(spec, 'spec.dataSchema.dataSource')) {
return 'missing spec.dataSchema.dataSource';
}
@@ -256,12 +259,12 @@ export interface DataSchema {
export type DimensionMode = 'specific' | 'auto-detect';
-export function getDimensionMode(spec: IngestionSpec): DimensionMode {
+export function getDimensionMode(spec: Partial<IngestionSpec>): DimensionMode {
const dimensions = deepGet(spec, 'spec.dataSchema.dimensionsSpec.dimensions') || EMPTY_ARRAY;
return Array.isArray(dimensions) && dimensions.length === 0 ? 'auto-detect' : 'specific';
}
-export function getRollup(spec: IngestionSpec): boolean {
+export function getRollup(spec: Partial<IngestionSpec>): boolean {
const specRollup = deepGet(spec, 'spec.dataSchema.granularitySpec.rollup');
return typeof specRollup === 'boolean' ? specRollup : true;
}
@@ -275,7 +278,7 @@ export function getSpecType(spec: Partial<IngestionSpec>): IngestionType {
);
}
-export function isTask(spec: IngestionSpec) {
+export function isTask(spec: Partial<IngestionSpec>) {
const type = String(getSpecType(spec));
return (
type.startsWith('index_') ||
@@ -283,7 +286,7 @@ export function isTask(spec: IngestionSpec) {
);
}
-export function isDruidSource(spec: IngestionSpec): boolean {
+export function isDruidSource(spec: Partial<IngestionSpec>): boolean {
return deepGet(spec, 'spec.ioConfig.inputSource.type') === 'druid';
}
@@ -318,7 +321,7 @@ export function normalizeSpec(spec: Partial<IngestionSpec>): IngestionSpec {
* Make sure that any extra junk in the spec other than 'type' and 'spec' is removed
* @param spec
*/
-export function cleanSpec(spec: IngestionSpec): IngestionSpec {
+export function cleanSpec(spec: Partial<IngestionSpec>): Partial<IngestionSpec> {
return {
type: spec.type,
spec: spec.spec,
@@ -585,7 +588,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
</p>
</>
),
- adjustment: (ioConfig: IoConfig) => {
+ adjustment: ioConfig => {
return deepSet(
ioConfig,
'inputSource.properties.secretAccessKey.type',
@@ -598,7 +601,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
label: 'Access key ID environment variable',
type: 'string',
placeholder: '(environment variable name)',
- defined: (ioConfig: IoConfig) =>
+ defined: ioConfig =>
deepGet(ioConfig, 'inputSource.properties.accessKeyId.type') === 'environment',
info: <p>The environment variable containing the S3 access key for this S3 bucket.</p>,
},
@@ -607,7 +610,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
label: 'Access key ID value',
type: 'string',
placeholder: '(access key)',
- defined: (ioConfig: IoConfig) =>
+ defined: ioConfig =>
deepGet(ioConfig, 'inputSource.properties.accessKeyId.type') === 'default',
info: (
<>
@@ -646,7 +649,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
label: 'Secret key value',
type: 'string',
placeholder: '(environment variable name)',
- defined: (ioConfig: IoConfig) =>
+ defined: ioConfig =>
deepGet(ioConfig, 'inputSource.properties.secretAccessKey.type') === 'environment',
info: <p>The environment variable containing the S3 secret key for this S3 bucket.</p>,
},
@@ -655,7 +658,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
label: 'Secret key value',
type: 'string',
placeholder: '(secret key)',
- defined: (ioConfig: IoConfig) =>
+ defined: ioConfig =>
deepGet(ioConfig, 'inputSource.properties.secretAccessKey.type') === 'default',
info: (
<>
@@ -825,7 +828,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
name: 'topic',
type: 'string',
required: true,
- defined: (i: IoConfig) => i.type === 'kafka',
+ defined: typeIs('kafka'),
},
{
name: 'consumerProperties',
@@ -1005,7 +1008,7 @@ export function getIoConfigTuningFormFields(
{
name: 'useEarliestOffset',
type: 'boolean',
- defined: (i: IoConfig) => i.type === 'kafka',
+ defined: typeIs('kafka'),
required: true,
info: (
<>
@@ -1021,7 +1024,7 @@ export function getIoConfigTuningFormFields(
{
name: 'useEarliestSequenceNumber',
type: 'boolean',
- defined: (i: IoConfig) => i.type === 'kinesis',
+ defined: typeIs('kinesis'),
required: true,
info: (
<>
@@ -1092,14 +1095,14 @@ export function getIoConfigTuningFormFields(
name: 'recordsPerFetch',
type: 'number',
defaultValue: 2000,
- defined: (i: IoConfig) => i.type === 'kinesis',
+ defined: typeIs('kinesis'),
info: <>The number of records to request per GetRecords call to Kinesis.</>,
},
{
name: 'pollTimeout',
type: 'number',
defaultValue: 100,
- defined: (i: IoConfig) => i.type === 'kafka',
+ defined: typeIs('kafka'),
info: (
<>
<p>
@@ -1112,14 +1115,14 @@ export function getIoConfigTuningFormFields(
name: 'fetchDelayMillis',
type: 'number',
defaultValue: 1000,
- defined: (i: IoConfig) => i.type === 'kinesis',
+ defined: typeIs('kinesis'),
info: <>Time in milliseconds to wait between subsequent GetRecords calls to Kinesis.</>,
},
{
name: 'deaggregate',
type: 'boolean',
defaultValue: false,
- defined: (i: IoConfig) => i.type === 'kinesis',
+ defined: typeIs('kinesis'),
info: <>Whether to use the de-aggregate function of the KCL.</>,
},
{
@@ -1187,7 +1190,7 @@ export function getIoConfigTuningFormFields(
name: 'skipOffsetGaps',
type: 'boolean',
defaultValue: false,
- defined: (i: IoConfig) => i.type === 'kafka',
+ defined: typeIs('kafka'),
info: (
<>
<p>
@@ -1221,13 +1224,13 @@ function basenameFromFilename(filename: string): string | undefined {
return filename.split('.')[0];
}
-export function fillDataSourceNameIfNeeded(spec: IngestionSpec): IngestionSpec {
+export function fillDataSourceNameIfNeeded(spec: Partial<IngestionSpec>): Partial<IngestionSpec> {
const possibleName = guessDataSourceName(spec);
if (!possibleName) return spec;
return deepSetIfUnset(spec, 'spec.dataSchema.dataSource', possibleName);
}
-export function guessDataSourceName(spec: IngestionSpec): string | undefined {
+export function guessDataSourceName(spec: Partial<IngestionSpec>): string | undefined {
const ioConfig = deepGet(spec, 'spec.ioConfig');
if (!ioConfig) return;
@@ -1331,7 +1334,7 @@ export interface PartitionsSpec {
assumeGrouped?: boolean;
}
-export function adjustForceGuaranteedRollup(spec: IngestionSpec) {
+export function adjustForceGuaranteedRollup(spec: Partial<IngestionSpec>) {
if (getSpecType(spec) !== 'index_parallel') return spec;
const partitionsSpecType = deepGet(spec, 'spec.tuningConfig.partitionsSpec.type') || 'dynamic';
@@ -1344,12 +1347,12 @@ export function adjustForceGuaranteedRollup(spec: IngestionSpec) {
return spec;
}
-export function invalidPartitionConfig(spec: IngestionSpec): boolean {
+export function invalidPartitionConfig(spec: Partial<IngestionSpec>): boolean {
return (
// Bad primary partitioning, or...
!deepGet(spec, 'spec.dataSchema.granularitySpec.segmentGranularity') ||
// Bad secondary partitioning
- Boolean(AutoForm.issueWithModel(spec, getSecondaryPartitionRelatedFormFields(spec, undefined)))
+ !AutoForm.isValidModel(spec, getSecondaryPartitionRelatedFormFields(spec, undefined))
);
}
@@ -1397,7 +1400,7 @@ export const PRIMARY_PARTITION_RELATED_FORM_FIELDS: Field<IngestionSpec>[] = [
];
export function getSecondaryPartitionRelatedFormFields(
- spec: IngestionSpec,
+ spec: Partial<IngestionSpec>,
dimensionSuggestions: string[] | undefined,
): Field<IngestionSpec>[] {
const specType = getSpecType(spec);
@@ -1591,7 +1594,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
type: 'number',
defaultValue: 1,
min: 1,
- defined: s => s.type === 'index_parallel',
+ defined: typeIs('index_parallel'),
info: (
<>
Maximum number of tasks which can be run at the same time. The supervisor task would spawn
@@ -1606,7 +1609,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
name: 'spec.tuningConfig.maxRetry',
type: 'number',
defaultValue: 3,
- defined: s => s.type === 'index_parallel',
+ defined: typeIs('index_parallel'),
hideInMore: true,
info: <>Maximum number of retries on task failures.</>,
},
@@ -1614,7 +1617,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
name: 'spec.tuningConfig.taskStatusCheckPeriodMs',
type: 'number',
defaultValue: 1000,
- defined: s => s.type === 'index_parallel',
+ defined: typeIs('index_parallel'),
hideInMore: true,
info: <>Polling period in milliseconds to check running task statuses.</>,
},
@@ -1661,7 +1664,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
name: 'spec.tuningConfig.resetOffsetAutomatically',
type: 'boolean',
defaultValue: false,
- defined: s => oneOf(s.type, 'kafka', 'kinesis'),
+ defined: typeIs('kafka', 'kinesis'),
info: (
<>
Whether to reset the consumer offset if the next offset that it is trying to fetch is less
@@ -1673,7 +1676,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
name: 'spec.tuningConfig.skipSequenceNumberAvailabilityCheck',
type: 'boolean',
defaultValue: false,
- defined: s => s.type === 'kinesis',
+ defined: typeIs('kinesis'),
info: (
<>
Whether to enable checking if the current sequence number is still available in a particular
@@ -1686,14 +1689,14 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
name: 'spec.tuningConfig.intermediatePersistPeriod',
type: 'duration',
defaultValue: 'PT10M',
- defined: s => oneOf(s.type, 'kafka', 'kinesis'),
+ defined: typeIs('kafka', 'kinesis'),
info: <>The period that determines the rate at which intermediate persists occur.</>,
},
{
name: 'spec.tuningConfig.intermediateHandoffPeriod',
type: 'duration',
defaultValue: 'P2147483647D',
- defined: s => oneOf(s.type, 'kafka', 'kinesis'),
+ defined: typeIs('kafka', 'kinesis'),
info: (
<>
How often the tasks should hand off segments. Handoff will happen either if
@@ -1730,7 +1733,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
name: 'spec.tuningConfig.handoffConditionTimeout',
type: 'number',
defaultValue: 0,
- defined: s => oneOf(s.type, 'kafka', 'kinesis'),
+ defined: typeIs('kafka', 'kinesis'),
hideInMore: true,
info: <>Milliseconds to wait for segment handoff. 0 means to wait forever.</>,
},
@@ -1799,7 +1802,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
type: 'number',
defaultValue: 1000,
min: 1,
- defined: s => s.type === 'index_parallel',
+ defined: typeIs('index_parallel'),
hideInMore: true,
adjustment: s => deepSet(s, 'splitHintSpec.type', 'maxSize'),
info: (
@@ -1817,7 +1820,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
name: 'spec.tuningConfig.chatHandlerTimeout',
type: 'duration',
defaultValue: 'PT10S',
- defined: s => s.type === 'index_parallel',
+ defined: typeIs('index_parallel'),
hideInMore: true,
info: <>Timeout for reporting the pushed segments in worker tasks.</>,
},
@@ -1825,7 +1828,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
name: 'spec.tuningConfig.chatHandlerNumRetries',
type: 'number',
defaultValue: 5,
- defined: s => s.type === 'index_parallel',
+ defined: typeIs('index_parallel'),
hideInMore: true,
info: <>Retries for reporting the pushed segments in worker tasks.</>,
},
@@ -1833,7 +1836,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
name: 'spec.tuningConfig.workerThreads',
type: 'number',
placeholder: 'min(10, taskCount)',
- defined: s => oneOf(s.type, 'kafka', 'kinesis'),
+ defined: typeIs('kafka', 'kinesis'),
info: (
<>The number of threads that will be used by the supervisor for asynchronous operations.</>
),
@@ -1842,7 +1845,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
name: 'spec.tuningConfig.chatThreads',
type: 'number',
placeholder: 'min(10, taskCount * replicas)',
- defined: s => oneOf(s.type, 'kafka', 'kinesis'),
+ defined: typeIs('kafka', 'kinesis'),
hideInMore: true,
info: <>The number of threads that will be used for communicating with indexing tasks.</>,
},
@@ -1850,7 +1853,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
name: 'spec.tuningConfig.chatRetries',
type: 'number',
defaultValue: 8,
- defined: s => oneOf(s.type, 'kafka', 'kinesis'),
+ defined: typeIs('kafka', 'kinesis'),
hideInMore: true,
info: (
<>
@@ -1863,14 +1866,14 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
name: 'spec.tuningConfig.httpTimeout',
type: 'duration',
defaultValue: 'PT10S',
- defined: s => oneOf(s.type, 'kafka', 'kinesis'),
+ defined: typeIs('kafka', 'kinesis'),
info: <>How long to wait for a HTTP response from an indexing task.</>,
},
{
name: 'spec.tuningConfig.shutdownTimeout',
type: 'duration',
defaultValue: 'PT80S',
- defined: s => oneOf(s.type, 'kafka', 'kinesis'),
+ defined: typeIs('kafka', 'kinesis'),
hideInMore: true,
info: (
<>
@@ -1882,7 +1885,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
name: 'spec.tuningConfig.offsetFetchPeriod',
type: 'duration',
defaultValue: 'PT30S',
- defined: s => s.type === 'kafka',
+ defined: typeIs('kafka'),
info: (
<>
How often the supervisor queries Kafka and the indexing tasks to fetch current offsets and
@@ -1894,7 +1897,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
name: 'spec.tuningConfig.recordBufferSize',
type: 'number',
defaultValue: 10000,
- defined: s => s.type === 'kinesis',
+ defined: typeIs('kinesis'),
info: (
<>
Size of the buffer (number of events) used between the Kinesis fetch threads and the main
@@ -1906,7 +1909,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
name: 'spec.tuningConfig.recordBufferOfferTimeout',
type: 'number',
defaultValue: 5000,
- defined: s => s.type === 'kinesis',
+ defined: typeIs('kinesis'),
hideInMore: true,
info: (
<>
@@ -1920,7 +1923,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
hideInMore: true,
type: 'number',
defaultValue: 5000,
- defined: s => s.type === 'kinesis',
+ defined: typeIs('kinesis'),
info: (
<>
Length of time in milliseconds to wait for the buffer to drain before attempting to fetch
@@ -1932,7 +1935,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
name: 'spec.tuningConfig.fetchSequenceNumberTimeout',
type: 'number',
defaultValue: 60000,
- defined: s => s.type === 'kinesis',
+ defined: typeIs('kinesis'),
hideInMore: true,
info: (
<>
@@ -1947,7 +1950,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
name: 'spec.tuningConfig.fetchThreads',
type: 'number',
placeholder: 'max(1, {numProcessors} - 1)',
- defined: s => s.type === 'kinesis',
+ defined: typeIs('kinesis'),
hideInMore: true,
info: (
<>
@@ -1960,7 +1963,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
name: 'spec.tuningConfig.maxRecordsPerPoll',
type: 'number',
defaultValue: 100,
- defined: s => s.type === 'kinesis',
+ defined: typeIs('kinesis'),
hideInMore: true,
info: (
<>
@@ -1973,7 +1976,7 @@ const TUNING_FORM_FIELDS: Field<IngestionSpec>[] = [
name: 'spec.tuningConfig.repartitionTransitionDuration',
type: 'duration',
defaultValue: 'PT2M',
- defined: s => s.type === 'kinesis',
+ defined: typeIs('kinesis'),
hideInMore: true,
info: (
<>
@@ -2013,9 +2016,9 @@ export interface Bitmap {
// --------------
export function updateIngestionType(
- spec: IngestionSpec,
+ spec: Partial<IngestionSpec>,
comboType: IngestionComboType,
-): IngestionSpec {
+): Partial<IngestionSpec> {
const [ingestionType, inputSourceType] = comboType.split(':');
const ioAndTuningConfigType = ingestionTypeToIoAndTuningConfigType(
ingestionType as IngestionType,
@@ -2061,7 +2064,10 @@ export function issueWithSampleData(sampleData: string[]): JSX.Element | undefin
return;
}
-export function fillInputFormatIfNeeded(spec: IngestionSpec, sampleData: string[]): IngestionSpec {
+export function fillInputFormatIfNeeded(
+ spec: Partial<IngestionSpec>,
+ sampleData: string[],
+): Partial<IngestionSpec> {
if (deepGet(spec, 'spec.ioConfig.inputFormat.type')) return spec;
return deepSet(spec, 'spec.ioConfig.inputFormat', guessInputFormat(sampleData));
}
@@ -2106,15 +2112,15 @@ export function guessInputFormat(sampleData: string[]): InputFormat {
}
function inputFormatFromType(type: string, findColumnsFromHeader?: boolean): InputFormat {
- const inputFormat: InputFormat = { type };
+ let inputFormat: InputFormat = { type };
if (type === 'regex') {
- inputFormat.pattern = '(.*)';
- inputFormat.columns = ['column1'];
+ inputFormat = deepSet(inputFormat, 'pattern', '(.*)');
+ inputFormat = deepSet(inputFormat, 'columns', ['column1']);
}
if (typeof findColumnsFromHeader === 'boolean') {
- inputFormat.findColumnsFromHeader = findColumnsFromHeader;
+ inputFormat = deepSet(inputFormat, 'findColumnsFromHeader', findColumnsFromHeader);
}
return inputFormat;
@@ -2147,7 +2153,7 @@ export function getColumnTypeFromHeaderAndRows(
);
}
-function getTypeHintsFromSpec(spec: IngestionSpec): Record<string, string> {
+function getTypeHintsFromSpec(spec: Partial<IngestionSpec>): Record<string, string> {
const typeHints: Record<string, string> = {};
const currentDimensions = deepGet(spec, 'spec.dataSchema.dimensionsSpec.dimensions') || [];
for (const currentDimension of currentDimensions) {
@@ -2167,12 +2173,12 @@ function getTypeHintsFromSpec(spec: IngestionSpec): Record<string, string> {
}
export function updateSchemaWithSample(
- spec: IngestionSpec,
+ spec: Partial<IngestionSpec>,
headerAndRows: HeaderAndRows,
dimensionMode: DimensionMode,
rollup: boolean,
forcePartitionInitialization = false,
-): IngestionSpec {
+): Partial<IngestionSpec> {
const typeHints = getTypeHintsFromSpec(spec);
let newSpec = spec;
@@ -2220,7 +2226,7 @@ export function updateSchemaWithSample(
// ------------------------
-export function upgradeSpec(spec: any): any {
+export function upgradeSpec(spec: any): Partial<IngestionSpec> {
if (deepGet(spec, 'spec.ioConfig.firehose')) {
switch (deepGet(spec, 'spec.ioConfig.firehose.type')) {
case 'static-s3':
@@ -2251,7 +2257,7 @@ export function upgradeSpec(spec: any): any {
return spec;
}
-export function downgradeSpec(spec: any): any {
+export function downgradeSpec(spec: Partial<IngestionSpec>): Partial<IngestionSpec> {
if (deepGet(spec, 'spec.ioConfig.inputSource')) {
spec = deepMove(spec, 'spec.ioConfig.inputFormat.type', 'spec.ioConfig.inputFormat.format');
spec = deepSet(spec, 'spec.dataSchema.parser', { type: 'string' });
diff --git a/web-console/src/druid-models/input-format.tsx b/web-console/src/druid-models/input-format.tsx
index 7c2f324..6f36eed 100644
--- a/web-console/src/druid-models/input-format.tsx
+++ b/web-console/src/druid-models/input-format.tsx
@@ -21,20 +21,20 @@ import React from 'react';
import { AutoForm, ExternalLink, Field } from '../components';
import { getLink } from '../links';
-import { oneOf } from '../utils';
+import { oneOf, typeIs } from '../utils';
import { FlattenSpec } from './flatten-spec';
export interface InputFormat {
- type: string;
- findColumnsFromHeader?: boolean;
- skipHeaderRows?: number;
- columns?: string[];
- listDelimiter?: string;
- pattern?: string;
- function?: string;
- flattenSpec?: FlattenSpec;
- keepNullColumns?: boolean;
+ readonly type: string;
+ readonly findColumnsFromHeader?: boolean;
+ readonly skipHeaderRows?: number;
+ readonly columns?: string[];
+ readonly listDelimiter?: string;
+ readonly pattern?: string;
+ readonly function?: string;
+ readonly flattenSpec?: FlattenSpec;
+ readonly keepNullColumns?: boolean;
}
export const INPUT_FORMAT_FIELDS: Field<InputFormat>[] = [
@@ -60,20 +60,20 @@ export const INPUT_FORMAT_FIELDS: Field<InputFormat>[] = [
{
name: 'pattern',
type: 'string',
+ defined: typeIs('regex'),
required: true,
- defined: (p: InputFormat) => p.type === 'regex',
},
{
name: 'function',
type: 'string',
+ defined: typeIs('javascript'),
required: true,
- defined: (p: InputFormat) => p.type === 'javascript',
},
{
name: 'skipHeaderRows',
type: 'number',
defaultValue: 0,
- defined: (p: InputFormat) => oneOf(p.type, 'csv', 'tsv'),
+ defined: typeIs('csv', 'tsv'),
min: 0,
info: (
<>
@@ -84,8 +84,8 @@ export const INPUT_FORMAT_FIELDS: Field<InputFormat>[] = [
{
name: 'findColumnsFromHeader',
type: 'boolean',
+ defined: typeIs('csv', 'tsv'),
required: true,
- defined: (p: InputFormat) => oneOf(p.type, 'csv', 'tsv'),
info: (
<>
If this is set, find the column names from the header row. Note that
@@ -100,7 +100,7 @@ export const INPUT_FORMAT_FIELDS: Field<InputFormat>[] = [
name: 'columns',
type: 'string-array',
required: true,
- defined: (p: InputFormat) =>
+ defined: p =>
(oneOf(p.type, 'csv', 'tsv') && p.findColumnsFromHeader === false) || p.type === 'regex',
info: (
<>
@@ -114,7 +114,7 @@ export const INPUT_FORMAT_FIELDS: Field<InputFormat>[] = [
type: 'string',
defaultValue: '\t',
suggestions: ['\t', '|', '#'],
- defined: (p: InputFormat) => p.type === 'tsv',
+ defined: typeIs('tsv'),
info: <>A custom delimiter for data values.</>,
},
{
@@ -122,14 +122,14 @@ export const INPUT_FORMAT_FIELDS: Field<InputFormat>[] = [
type: 'string',
defaultValue: '\x01',
suggestions: ['\x01', '\x00'],
- defined: (p: InputFormat) => oneOf(p.type, 'csv', 'tsv', 'regex'),
+ defined: typeIs('csv', 'tsv', 'regex'),
info: <>A custom delimiter for multi-value dimensions.</>,
},
{
name: 'binaryAsString',
type: 'boolean',
defaultValue: false,
- defined: (p: InputFormat) => oneOf(p.type, 'parquet', 'orc', 'avro_ocf', 'avro_stream'),
+ defined: typeIs('parquet', 'orc', 'avro_ocf', 'avro_stream'),
info: (
<>
Specifies if the binary column which is not logically marked as a string should be treated
@@ -143,6 +143,10 @@ export function issueWithInputFormat(inputFormat: InputFormat | undefined): stri
return AutoForm.issueWithModel(inputFormat, INPUT_FORMAT_FIELDS);
}
-export function inputFormatCanFlatten(inputFormat: InputFormat): boolean {
- return oneOf(inputFormat.type, 'json', 'parquet', 'orc', 'avro_ocf', 'avro_stream');
-}
+export const inputFormatCanFlatten: (inputFormat: InputFormat) => boolean = typeIs(
+ 'json',
+ 'parquet',
+ 'orc',
+ 'avro_ocf',
+ 'avro_stream',
+);
diff --git a/web-console/src/druid-models/lookup-spec.tsx b/web-console/src/druid-models/lookup-spec.tsx
index 0b621fa..c8eee20 100644
--- a/web-console/src/druid-models/lookup-spec.tsx
+++ b/web-console/src/druid-models/lookup-spec.tsx
@@ -20,47 +20,47 @@ import { Code } from '@blueprintjs/core';
import React from 'react';
import { AutoForm, Field } from '../components';
-import { deepGet, deepSet, oneOf } from '../utils';
+import { deepGet, deepSet, oneOf, typeIs } from '../utils';
export interface ExtractionNamespaceSpec {
- type?: string;
- uri?: string;
- uriPrefix?: string;
- fileRegex?: string;
- namespaceParseSpec?: NamespaceParseSpec;
- connectorConfig?: {
- createTables: boolean;
- connectURI: string;
- user: string;
- password: string;
+ readonly type: string;
+ readonly uri?: string;
+ readonly uriPrefix?: string;
+ readonly fileRegex?: string;
+ readonly namespaceParseSpec?: NamespaceParseSpec;
+ readonly connectorConfig?: {
+ readonly createTables: boolean;
+ readonly connectURI: string;
+ readonly user: string;
+ readonly password: string;
};
- table?: string;
- keyColumn?: string;
- valueColumn?: string;
- filter?: any;
- tsColumn?: string;
- pollPeriod?: number | string;
+ readonly table?: string;
+ readonly keyColumn?: string;
+ readonly valueColumn?: string;
+ readonly filter?: any;
+ readonly tsColumn?: string;
+ readonly pollPeriod?: number | string;
}
export interface NamespaceParseSpec {
- format: string;
- columns?: string[];
- keyColumn?: string;
- valueColumn?: string;
- hasHeaderRow?: boolean;
- skipHeaderRows?: number;
- keyFieldName?: string;
- valueFieldName?: string;
- delimiter?: string;
- listDelimiter?: string;
+ readonly format: string;
+ readonly columns?: string[];
+ readonly keyColumn?: string;
+ readonly valueColumn?: string;
+ readonly hasHeaderRow?: boolean;
+ readonly skipHeaderRows?: number;
+ readonly keyFieldName?: string;
+ readonly valueFieldName?: string;
+ readonly delimiter?: string;
+ readonly listDelimiter?: string;
}
export interface LookupSpec {
- type?: string;
- map?: Record<string, string | number>;
- extractionNamespace?: ExtractionNamespaceSpec;
- firstCacheTimeout?: number;
- injective?: boolean;
+ readonly type: string;
+ readonly map?: Record<string, string | number>;
+ readonly extractionNamespace?: ExtractionNamespaceSpec;
+ readonly firstCacheTimeout?: number;
+ readonly injective?: boolean;
}
export const LOOKUP_FIELDS: Field<LookupSpec>[] = [
@@ -69,14 +69,14 @@ export const LOOKUP_FIELDS: Field<LookupSpec>[] = [
type: 'string',
suggestions: ['map', 'cachedNamespace'],
required: true,
- adjustment: (model: LookupSpec) => {
- if (model.type === 'map' && !model.map) {
- return deepSet(model, 'map', {});
+ adjustment: l => {
+ if (l.type === 'map' && !l.map) {
+ return deepSet(l, 'map', {});
}
- if (model.type === 'cachedNamespace' && !deepGet(model, 'extractionNamespace.type')) {
- return deepSet(model, 'extractionNamespace', { type: 'uri' });
+ if (l.type === 'cachedNamespace' && !deepGet(l, 'extractionNamespace.type')) {
+ return deepSet(l, 'extractionNamespace', { type: 'uri' });
}
- return model;
+ return l;
},
},
@@ -85,7 +85,7 @@ export const LOOKUP_FIELDS: Field<LookupSpec>[] = [
name: 'map',
type: 'json',
height: '60vh',
- defined: (model: LookupSpec) => model.type === 'map',
+ defined: typeIs('map'),
required: true,
issueWithValue: value => {
if (!value) return 'map must be defined';
@@ -107,7 +107,7 @@ export const LOOKUP_FIELDS: Field<LookupSpec>[] = [
type: 'string',
placeholder: 'uri',
suggestions: ['uri', 'jdbc'],
- defined: (model: LookupSpec) => model.type === 'cachedNamespace',
+ defined: typeIs('cachedNamespace'),
required: true,
},
{
@@ -115,12 +115,10 @@ export const LOOKUP_FIELDS: Field<LookupSpec>[] = [
label: 'URI prefix',
type: 'string',
placeholder: 's3://bucket/some/key/prefix/',
- defined: (model: LookupSpec) =>
- deepGet(model, 'extractionNamespace.type') === 'uri' &&
- !deepGet(model, 'extractionNamespace.uri'),
- required: (model: LookupSpec) =>
- !deepGet(model, 'extractionNamespace.uriPrefix') &&
- !deepGet(model, 'extractionNamespace.uri'),
+ defined: l =>
+ deepGet(l, 'extractionNamespace.type') === 'uri' && !deepGet(l, 'extractionNamespace.uri'),
+ required: l =>
+ !deepGet(l, 'extractionNamespace.uriPrefix') && !deepGet(l, 'extractionNamespace.uri'),
info:
'A URI which specifies a directory (or other searchable resource) in which to search for files',
},
@@ -129,12 +127,11 @@ export const LOOKUP_FIELDS: Field<LookupSpec>[] = [
type: 'string',
label: 'URI (deprecated)',
placeholder: 's3://bucket/some/key/prefix/lookups-01.gz',
- defined: (model: LookupSpec) =>
- deepGet(model, 'extractionNamespace.type') === 'uri' &&
- !deepGet(model, 'extractionNamespace.uriPrefix'),
- required: (model: LookupSpec) =>
- !deepGet(model, 'extractionNamespace.uriPrefix') &&
- !deepGet(model, 'extractionNamespace.uri'),
+ defined: l =>
+ deepGet(l, 'extractionNamespace.type') === 'uri' &&
+ !deepGet(l, 'extractionNamespace.uriPrefix'),
+ required: l =>
+ !deepGet(l, 'extractionNamespace.uriPrefix') && !deepGet(l, 'extractionNamespace.uri'),
info: (
<>
<p>URI for the file of interest, specified as a file, hdfs, or s3 path</p>
@@ -147,9 +144,9 @@ export const LOOKUP_FIELDS: Field<LookupSpec>[] = [
label: 'File regex',
type: 'string',
defaultValue: '.*',
- defined: (model: LookupSpec) =>
- deepGet(model, 'extractionNamespace.type') === 'uri' &&
- Boolean(deepGet(model, 'extractionNamespace.uriPrefix')),
+ defined: l =>
+ deepGet(l, 'extractionNamespace.type') === 'uri' &&
+ Boolean(deepGet(l, 'extractionNamespace.uriPrefix')),
info: 'Optional regex for matching the file name under uriPrefix.',
},
@@ -159,7 +156,7 @@ export const LOOKUP_FIELDS: Field<LookupSpec>[] = [
label: 'Parse format',
type: 'string',
suggestions: ['csv', 'tsv', 'simpleJson', 'customJson'],
- defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'uri',
+ defined: l => deepGet(l, 'extractionNamespace.type') === 'uri',
required: true,
info: (
<>
@@ -178,47 +175,46 @@ export const LOOKUP_FIELDS: Field<LookupSpec>[] = [
name: 'extractionNamespace.namespaceParseSpec.skipHeaderRows',
type: 'number',
defaultValue: 0,
- defined: (model: LookupSpec) =>
- deepGet(model, 'extractionNamespace.type') === 'uri' &&
- oneOf(deepGet(model, 'extractionNamespace.namespaceParseSpec.format'), 'csv', 'tsv'),
+ defined: l =>
+ deepGet(l, 'extractionNamespace.type') === 'uri' &&
+ oneOf(deepGet(l, 'extractionNamespace.namespaceParseSpec.format'), 'csv', 'tsv'),
info: `Number of header rows to be skipped. The default number of header rows to be skipped is 0.`,
},
{
name: 'extractionNamespace.namespaceParseSpec.hasHeaderRow',
type: 'boolean',
defaultValue: false,
- defined: (model: LookupSpec) =>
- deepGet(model, 'extractionNamespace.type') === 'uri' &&
- oneOf(deepGet(model, 'extractionNamespace.namespaceParseSpec.format'), 'csv', 'tsv'),
+ defined: l =>
+ deepGet(l, 'extractionNamespace.type') === 'uri' &&
+ oneOf(deepGet(l, 'extractionNamespace.namespaceParseSpec.format'), 'csv', 'tsv'),
info: `A flag to indicate that column information can be extracted from the input files' header row`,
},
{
name: 'extractionNamespace.namespaceParseSpec.columns',
type: 'string-array',
placeholder: `["key", "value"]`,
- defined: (model: LookupSpec) =>
- deepGet(model, 'extractionNamespace.type') === 'uri' &&
- oneOf(deepGet(model, 'extractionNamespace.namespaceParseSpec.format'), 'csv', 'tsv'),
- required: (model: LookupSpec) =>
- !deepGet(model, 'extractionNamespace.namespaceParseSpec.hasHeaderRow'),
+ defined: l =>
+ deepGet(l, 'extractionNamespace.type') === 'uri' &&
+ oneOf(deepGet(l, 'extractionNamespace.namespaceParseSpec.format'), 'csv', 'tsv'),
+ required: l => !deepGet(l, 'extractionNamespace.namespaceParseSpec.hasHeaderRow'),
info: 'The list of columns in the csv file',
},
{
name: 'extractionNamespace.namespaceParseSpec.keyColumn',
type: 'string',
placeholder: '(optional - defaults to the first column)',
- defined: (model: LookupSpec) =>
- deepGet(model, 'extractionNamespace.type') === 'uri' &&
- oneOf(deepGet(model, 'extractionNamespace.namespaceParseSpec.format'), 'csv', 'tsv'),
+ defined: l =>
+ deepGet(l, 'extractionNamespace.type') === 'uri' &&
+ oneOf(deepGet(l, 'extractionNamespace.namespaceParseSpec.format'), 'csv', 'tsv'),
info: 'The name of the column containing the key',
},
{
name: 'extractionNamespace.namespaceParseSpec.valueColumn',
type: 'string',
placeholder: '(optional - defaults to the second column)',
- defined: (model: LookupSpec) =>
- deepGet(model, 'extractionNamespace.type') === 'uri' &&
- oneOf(deepGet(model, 'extractionNamespace.namespaceParseSpec.format'), 'csv', 'tsv'),
+ defined: l =>
+ deepGet(l, 'extractionNamespace.type') === 'uri' &&
+ oneOf(deepGet(l, 'extractionNamespace.namespaceParseSpec.format'), 'csv', 'tsv'),
info: 'The name of the column containing the value',
},
@@ -227,17 +223,17 @@ export const LOOKUP_FIELDS: Field<LookupSpec>[] = [
name: 'extractionNamespace.namespaceParseSpec.delimiter',
type: 'string',
placeholder: `(optional)`,
- defined: (model: LookupSpec) =>
- deepGet(model, 'extractionNamespace.type') === 'uri' &&
- deepGet(model, 'extractionNamespace.namespaceParseSpec.format') === 'tsv',
+ defined: l =>
+ deepGet(l, 'extractionNamespace.type') === 'uri' &&
+ deepGet(l, 'extractionNamespace.namespaceParseSpec.format') === 'tsv',
},
{
name: 'extractionNamespace.namespaceParseSpec.listDelimiter',
type: 'string',
placeholder: `(optional)`,
- defined: (model: LookupSpec) =>
- deepGet(model, 'extractionNamespace.type') === 'uri' &&
- deepGet(model, 'extractionNamespace.namespaceParseSpec.format') === 'tsv',
+ defined: l =>
+ deepGet(l, 'extractionNamespace.type') === 'uri' &&
+ deepGet(l, 'extractionNamespace.namespaceParseSpec.format') === 'tsv',
},
// Custom JSON
@@ -245,26 +241,25 @@ export const LOOKUP_FIELDS: Field<LookupSpec>[] = [
name: 'extractionNamespace.namespaceParseSpec.keyFieldName',
type: 'string',
placeholder: `key`,
- defined: (model: LookupSpec) =>
- deepGet(model, 'extractionNamespace.type') === 'uri' &&
- deepGet(model, 'extractionNamespace.namespaceParseSpec.format') === 'customJson',
+ defined: l =>
+ deepGet(l, 'extractionNamespace.type') === 'uri' &&
+ deepGet(l, 'extractionNamespace.namespaceParseSpec.format') === 'customJson',
required: true,
},
{
name: 'extractionNamespace.namespaceParseSpec.valueFieldName',
type: 'string',
placeholder: `value`,
- defined: (model: LookupSpec) =>
- deepGet(model, 'extractionNamespace.type') === 'uri' &&
- deepGet(model, 'extractionNamespace.namespaceParseSpec.format') === 'customJson',
+ defined: l =>
+ deepGet(l, 'extractionNamespace.type') === 'uri' &&
+ deepGet(l, 'extractionNamespace.namespaceParseSpec.format') === 'customJson',
required: true,
},
{
name: 'extractionNamespace.pollPeriod',
type: 'string',
defaultValue: '0',
- defined: (model: LookupSpec) =>
- oneOf(deepGet(model, 'extractionNamespace.type'), 'uri', 'jdbc'),
+ defined: l => oneOf(deepGet(l, 'extractionNamespace.type'), 'uri', 'jdbc'),
info: `Period between polling for updates`,
},
@@ -273,33 +268,33 @@ export const LOOKUP_FIELDS: Field<LookupSpec>[] = [
name: 'extractionNamespace.connectorConfig.connectURI',
label: 'Connect URI',
type: 'string',
- defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'jdbc',
+ defined: l => deepGet(l, 'extractionNamespace.type') === 'jdbc',
required: true,
info: 'Defines the connectURI value on the The connector config to used',
},
{
name: 'extractionNamespace.connectorConfig.user',
type: 'string',
- defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'jdbc',
+ defined: l => deepGet(l, 'extractionNamespace.type') === 'jdbc',
info: 'Defines the user to be used by the connector config',
},
{
name: 'extractionNamespace.connectorConfig.password',
type: 'string',
- defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'jdbc',
+ defined: l => deepGet(l, 'extractionNamespace.type') === 'jdbc',
info: 'Defines the password to be used by the connector config',
},
{
name: 'extractionNamespace.connectorConfig.createTables',
type: 'boolean',
- defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'jdbc',
+ defined: l => deepGet(l, 'extractionNamespace.type') === 'jdbc',
info: 'Should tables be created',
},
{
name: 'extractionNamespace.table',
type: 'string',
placeholder: 'some_lookup_table',
- defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'jdbc',
+ defined: l => deepGet(l, 'extractionNamespace.type') === 'jdbc',
required: true,
info: (
<>
@@ -318,7 +313,7 @@ export const LOOKUP_FIELDS: Field<LookupSpec>[] = [
name: 'extractionNamespace.keyColumn',
type: 'string',
placeholder: 'my_key_value',
- defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'jdbc',
+ defined: l => deepGet(l, 'extractionNamespace.type') === 'jdbc',
required: true,
info: (
<>
@@ -337,7 +332,7 @@ export const LOOKUP_FIELDS: Field<LookupSpec>[] = [
name: 'extractionNamespace.valueColumn',
type: 'string',
placeholder: 'my_column_value',
- defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'jdbc',
+ defined: l => deepGet(l, 'extractionNamespace.type') === 'jdbc',
required: true,
info: (
<>
@@ -356,7 +351,7 @@ export const LOOKUP_FIELDS: Field<LookupSpec>[] = [
name: 'extractionNamespace.filter',
type: 'string',
placeholder: '(optional)',
- defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'jdbc',
+ defined: l => deepGet(l, 'extractionNamespace.type') === 'jdbc',
info: (
<>
<p>
@@ -375,7 +370,7 @@ export const LOOKUP_FIELDS: Field<LookupSpec>[] = [
type: 'string',
label: 'Timestamp column',
placeholder: '(optional)',
- defined: (model: LookupSpec) => deepGet(model, 'extractionNamespace.type') === 'jdbc',
+ defined: l => deepGet(l, 'extractionNamespace.type') === 'jdbc',
info: (
<>
<p>
@@ -395,14 +390,14 @@ export const LOOKUP_FIELDS: Field<LookupSpec>[] = [
name: 'firstCacheTimeout',
type: 'number',
defaultValue: 0,
- defined: (model: LookupSpec) => model.type === 'cachedNamespace',
+ defined: typeIs('cachedNamespace'),
info: `How long to wait (in ms) for the first run of the cache to populate. 0 indicates to not wait`,
},
{
name: 'injective',
type: 'boolean',
defaultValue: false,
- defined: (model: LookupSpec) => model.type === 'cachedNamespace',
+ defined: typeIs('cachedNamespace'),
info: `If the underlying map is injective (keys and values are unique) then optimizations can occur internally by setting this to true`,
},
];
@@ -411,12 +406,12 @@ export function isLookupInvalid(
lookupName: string | undefined,
lookupVersion: string | undefined,
lookupTier: string | undefined,
- lookupSpec: LookupSpec | undefined,
+ lookupSpec: Partial<LookupSpec>,
) {
return (
!lookupName ||
!lookupVersion ||
!lookupTier ||
- Boolean(AutoForm.issueWithModel(lookupSpec, LOOKUP_FIELDS))
+ !AutoForm.isValidModel(lookupSpec, LOOKUP_FIELDS)
);
}
diff --git a/web-console/src/druid-models/metric-spec.tsx b/web-console/src/druid-models/metric-spec.tsx
index e0c7f02..2e04665 100644
--- a/web-console/src/druid-models/metric-spec.tsx
+++ b/web-console/src/druid-models/metric-spec.tsx
@@ -21,38 +21,41 @@ import React from 'react';
import { ExternalLink, Field } from '../components';
import { getLink } from '../links';
-import { filterMap, oneOf } from '../utils';
+import { filterMap, typeIs } from '../utils';
import { HeaderAndRows } from '../utils/sampler';
import { getColumnTypeFromHeaderAndRows } from './ingestion-spec';
export interface MetricSpec {
- type: string;
- name?: string;
- fieldName?: string;
- maxStringBytes?: number;
- filterNullValues?: boolean;
- fieldNames?: string[];
- fnAggregate?: string;
- fnCombine?: string;
- fnReset?: string;
- fields?: string[];
- byRow?: boolean;
- round?: boolean;
- isInputHyperUnique?: boolean;
- filter?: any;
- aggregator?: MetricSpec;
+ readonly type: string;
+ readonly name?: string;
+ readonly fieldName?: string;
+ readonly maxStringBytes?: number;
+ readonly filterNullValues?: boolean;
+ readonly fieldNames?: string[];
+ readonly fnAggregate?: string;
+ readonly fnCombine?: string;
+ readonly fnReset?: string;
+ readonly fields?: string[];
+ readonly byRow?: boolean;
+ readonly round?: boolean;
+ readonly isInputHyperUnique?: boolean;
+ readonly filter?: any;
+ readonly aggregator?: MetricSpec;
}
export const METRIC_SPEC_FIELDS: Field<MetricSpec>[] = [
{
name: 'name',
type: 'string',
+ required: true,
info: <>The metric name as it will appear in Druid.</>,
+ placeholder: 'metric_name',
},
{
name: 'type',
type: 'string',
+ required: true,
suggestions: [
'count',
{
@@ -87,41 +90,58 @@ export const METRIC_SPEC_FIELDS: Field<MetricSpec>[] = [
{
name: 'fieldName',
type: 'string',
- defined: m => m.type !== 'filtered',
+ defined: typeIs(
+ 'longSum',
+ 'doubleSum',
+ 'floatSum',
+ 'longMin',
+ 'doubleMin',
+ 'floatMin',
+ 'longMax',
+ 'doubleMax',
+ 'floatMax',
+ 'thetaSketch',
+ 'HLLSketchBuild',
+ 'HLLSketchMerge',
+ 'quantilesDoublesSketch',
+ 'momentSketch',
+ 'fixedBucketsHistogram',
+ 'hyperUnique',
+ ),
+ required: true,
+ placeholder: 'column_name',
info: <>The column name for the aggregator to operate on.</>,
},
{
name: 'maxStringBytes',
type: 'number',
defaultValue: 1024,
- defined: m => {
- return oneOf(m.type, 'stringFirst', 'stringLast');
- },
+ defined: typeIs('stringFirst', 'stringLast'),
},
{
name: 'filterNullValues',
type: 'boolean',
defaultValue: false,
- defined: m => {
- return oneOf(m.type, 'stringFirst', 'stringLast');
- },
+ defined: typeIs('stringFirst', 'stringLast'),
},
// filtered
{
name: 'filter',
type: 'json',
- defined: m => m.type === 'filtered',
+ defined: typeIs('filtered'),
+ required: true,
},
{
name: 'aggregator',
type: 'json',
- defined: m => m.type === 'filtered',
+ defined: typeIs('filtered'),
+ required: true,
},
// thetaSketch
{
name: 'size',
type: 'number',
- defined: m => m.type === 'thetaSketch',
+ defined: typeIs('thetaSketch'),
defaultValue: 16384,
info: (
<>
@@ -145,7 +165,7 @@ export const METRIC_SPEC_FIELDS: Field<MetricSpec>[] = [
{
name: 'isInputThetaSketch',
type: 'boolean',
- defined: m => m.type === 'thetaSketch',
+ defined: typeIs('thetaSketch'),
defaultValue: false,
info: (
<>
@@ -159,7 +179,7 @@ export const METRIC_SPEC_FIELDS: Field<MetricSpec>[] = [
{
name: 'lgK',
type: 'number',
- defined: m => oneOf(m.type, 'HLLSketchBuild', 'HLLSketchMerge'),
+ defined: typeIs('HLLSketchBuild', 'HLLSketchMerge'),
defaultValue: 12,
info: (
<>
@@ -174,7 +194,7 @@ export const METRIC_SPEC_FIELDS: Field<MetricSpec>[] = [
{
name: 'tgtHllType',
type: 'string',
- defined: m => oneOf(m.type, 'HLLSketchBuild', 'HLLSketchMerge'),
+ defined: typeIs('HLLSketchBuild', 'HLLSketchMerge'),
defaultValue: 'HLL_4',
suggestions: ['HLL_4', 'HLL_6', 'HLL_8'],
info: (
@@ -188,7 +208,7 @@ export const METRIC_SPEC_FIELDS: Field<MetricSpec>[] = [
{
name: 'k',
type: 'number',
- defined: m => m.type === 'quantilesDoublesSketch',
+ defined: typeIs('quantilesDoublesSketch'),
defaultValue: 128,
info: (
<>
@@ -210,7 +230,7 @@ export const METRIC_SPEC_FIELDS: Field<MetricSpec>[] = [
{
name: 'k',
type: 'number',
- defined: m => m.type === 'momentSketch',
+ defined: typeIs('momentSketch'),
required: true,
info: (
<>
@@ -222,7 +242,7 @@ export const METRIC_SPEC_FIELDS: Field<MetricSpec>[] = [
{
name: 'compress',
type: 'boolean',
- defined: m => m.type === 'momentSketch',
+ defined: typeIs('momentSketch'),
defaultValue: true,
info: (
<>
@@ -236,21 +256,21 @@ export const METRIC_SPEC_FIELDS: Field<MetricSpec>[] = [
{
name: 'lowerLimit',
type: 'number',
- defined: m => m.type === 'fixedBucketsHistogram',
+ defined: typeIs('fixedBucketsHistogram'),
required: true,
info: <>Lower limit of the histogram.</>,
},
{
name: 'upperLimit',
type: 'number',
- defined: m => m.type === 'fixedBucketsHistogram',
+ defined: typeIs('fixedBucketsHistogram'),
required: true,
info: <>Upper limit of the histogram.</>,
},
{
name: 'numBuckets',
type: 'number',
- defined: m => m.type === 'fixedBucketsHistogram',
+ defined: typeIs('fixedBucketsHistogram'),
defaultValue: 10,
required: true,
info: (
@@ -263,7 +283,7 @@ export const METRIC_SPEC_FIELDS: Field<MetricSpec>[] = [
{
name: 'outlierHandlingMode',
type: 'string',
- defined: m => m.type === 'fixedBucketsHistogram',
+ defined: typeIs('fixedBucketsHistogram'),
required: true,
suggestions: ['ignore', 'overflow', 'clip'],
info: (
@@ -289,7 +309,7 @@ export const METRIC_SPEC_FIELDS: Field<MetricSpec>[] = [
{
name: 'isInputHyperUnique',
type: 'boolean',
- defined: m => m.type === 'hyperUnique',
+ defined: typeIs('hyperUnique'),
defaultValue: false,
info: (
<>
diff --git a/web-console/src/druid-models/timestamp-spec.tsx b/web-console/src/druid-models/timestamp-spec.tsx
index 6aa0486..c30bb0f 100644
--- a/web-console/src/druid-models/timestamp-spec.tsx
+++ b/web-console/src/druid-models/timestamp-spec.tsx
@@ -51,7 +51,7 @@ export const CONSTANT_TIMESTAMP_SPEC: TimestampSpec = {
export type TimestampSchema = 'none' | 'column' | 'expression';
-export function getTimestampSchema(spec: IngestionSpec): TimestampSchema {
+export function getTimestampSchema(spec: Partial<IngestionSpec>): TimestampSchema {
const transforms: Transform[] =
deepGet(spec, 'spec.dataSchema.transformSpec.transforms') || EMPTY_ARRAY;
@@ -63,21 +63,23 @@ export function getTimestampSchema(spec: IngestionSpec): TimestampSchema {
}
export interface TimestampSpec {
- column?: string;
- format?: string;
- missingValue?: string;
+ readonly column?: string;
+ readonly format?: string;
+ readonly missingValue?: string;
}
-export function getTimestampSpecColumnFromSpec(spec: IngestionSpec): string {
+export function getTimestampSpecColumnFromSpec(spec: Partial<IngestionSpec>): string {
// For the default https://github.com/apache/druid/blob/master/core/src/main/java/org/apache/druid/data/input/impl/TimestampSpec.java#L44
return deepGet(spec, 'spec.dataSchema.timestampSpec.column') || 'timestamp';
}
-export function getTimestampSpecConstantFromSpec(spec: IngestionSpec): string | undefined {
+export function getTimestampSpecConstantFromSpec(spec: Partial<IngestionSpec>): string | undefined {
return deepGet(spec, 'spec.dataSchema.timestampSpec.missingValue');
}
-export function getTimestampSpecExpressionFromSpec(spec: IngestionSpec): string | undefined {
+export function getTimestampSpecExpressionFromSpec(
+ spec: Partial<IngestionSpec>,
+): string | undefined {
const transforms: Transform[] =
deepGet(spec, 'spec.dataSchema.transformSpec.transforms') || EMPTY_ARRAY;
@@ -86,7 +88,7 @@ export function getTimestampSpecExpressionFromSpec(spec: IngestionSpec): string
return timeTransform.expression;
}
-export function getTimestampDetailFromSpec(spec: IngestionSpec): string {
+export function getTimestampDetailFromSpec(spec: Partial<IngestionSpec>): string {
const timestampSchema = getTimestampSchema(spec);
switch (timestampSchema) {
case 'none':
diff --git a/web-console/src/druid-models/transform-spec.tsx b/web-console/src/druid-models/transform-spec.tsx
index 2e84ecc..9d53761 100644
--- a/web-console/src/druid-models/transform-spec.tsx
+++ b/web-console/src/druid-models/transform-spec.tsx
@@ -25,14 +25,14 @@ import { getLink } from '../links';
import { TIME_COLUMN } from './timestamp-spec';
export interface TransformSpec {
- transforms?: Transform[];
- filter?: Record<string, any>;
+ readonly transforms?: Transform[];
+ readonly filter?: Record<string, any>;
}
export interface Transform {
- type: string;
- name: string;
- expression: string;
+ readonly type: string;
+ readonly name: string;
+ readonly expression: string;
}
export const TRANSFORM_FIELDS: Field<Transform>[] = [
diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx
index 1b0aa4f..9b5c9ce 100644
--- a/web-console/src/utils/general.tsx
+++ b/web-console/src/utils/general.tsx
@@ -154,6 +154,13 @@ export function oneOf<T>(thing: T, ...options: T[]): boolean {
return options.includes(thing);
}
+export function typeIs<T extends { type?: S }, S = string>(...options: S[]): (x: T) => boolean {
+ return x => {
+ if (x.type == null) return false;
+ return options.includes(x.type);
+ };
+}
+
// ----------------------------
export function countBy<T>(
@@ -227,11 +234,6 @@ export function uniq(array: readonly string[]): string[] {
});
}
-export function parseList(list: string): string[] {
- if (!list) return [];
- return list.split(',');
-}
-
// ----------------------------
export function formatInteger(n: NumberLike): string {
diff --git a/web-console/src/utils/sampler.ts b/web-console/src/utils/sampler.ts
index 4999182..c2f2ed9 100644
--- a/web-console/src/utils/sampler.ts
+++ b/web-console/src/utils/sampler.ts
@@ -101,12 +101,8 @@ function dedupe(xs: string[]): string[] {
});
}
-export function getCacheRowsFromSampleResponse(
- sampleResponse: SampleResponse,
- useParsed = false,
-): CacheRows {
- const key = useParsed ? 'parsed' : 'input';
- return filterMap(sampleResponse.data, d => d[key]).slice(0, 20);
+export function getCacheRowsFromSampleResponse(sampleResponse: SampleResponse): CacheRows {
+ return filterMap(sampleResponse.data, d => d.input).slice(0, 20);
}
export function applyCache(sampleSpec: SampleSpec, cacheRows: CacheRows) {
@@ -130,8 +126,10 @@ export function applyCache(sampleSpec: SampleSpec, cacheRows: CacheRows) {
});
const flattenSpec = deepGet(sampleSpec, 'spec.ioConfig.inputFormat.flattenSpec');
- const inputFormat: InputFormat = { type: 'json', keepNullColumns: true };
- if (flattenSpec) inputFormat.flattenSpec = flattenSpec;
+ let inputFormat: InputFormat = { type: 'json', keepNullColumns: true };
+ if (flattenSpec) {
+ inputFormat = deepSet(inputFormat, 'flattenSpec', flattenSpec);
+ }
sampleSpec = deepSet(sampleSpec, 'spec.ioConfig.inputFormat', inputFormat);
return sampleSpec;
@@ -259,7 +257,7 @@ function cleanupQueryGranularity(queryGranularity: any): any {
}
export async function sampleForConnect(
- spec: IngestionSpec,
+ spec: Partial<IngestionSpec>,
sampleStrategy: SampleStrategy,
): Promise<SampleResponseWithExtraInfo> {
const samplerType = getSpecType(spec);
@@ -324,7 +322,7 @@ export async function sampleForConnect(
}
export async function sampleForParser(
- spec: IngestionSpec,
+ spec: Partial<IngestionSpec>,
sampleStrategy: SampleStrategy,
): Promise<SampleResponse> {
const samplerType = getSpecType(spec);
@@ -353,7 +351,7 @@ export async function sampleForParser(
}
export async function sampleForTimestamp(
- spec: IngestionSpec,
+ spec: Partial<IngestionSpec>,
cacheRows: CacheRows,
): Promise<SampleResponse> {
const samplerType = getSpecType(spec);
@@ -425,7 +423,7 @@ export async function sampleForTimestamp(
}
export async function sampleForTransform(
- spec: IngestionSpec,
+ spec: Partial<IngestionSpec>,
cacheRows: CacheRows,
): Promise<SampleResponse> {
const samplerType = getSpecType(spec);
@@ -433,7 +431,7 @@ export async function sampleForTransform(
const transforms: Transform[] = deepGet(spec, 'spec.dataSchema.transformSpec.transforms') || [];
// Extra step to simulate auto detecting dimension with transforms
- const specialDimensionSpec: DimensionsSpec = {};
+ let specialDimensionSpec: DimensionsSpec = {};
if (transforms && transforms.length) {
const sampleSpecHack: SampleSpec = {
type: samplerType,
@@ -453,11 +451,15 @@ export async function sampleForTransform(
'transform-pre',
);
- specialDimensionSpec.dimensions = dedupe(
- headerFromSampleResponse({
- sampleResponse: sampleResponseHack,
- ignoreTimeColumn: true,
- }).concat(getDimensionNamesFromTransforms(transforms)),
+ specialDimensionSpec = deepSet(
+ specialDimensionSpec,
+ 'dimensions',
+ dedupe(
+ headerFromSampleResponse({
+ sampleResponse: sampleResponseHack,
+ ignoreTimeColumn: true,
+ }).concat(getDimensionNamesFromTransforms(transforms)),
+ ),
);
}
@@ -481,7 +483,7 @@ export async function sampleForTransform(
}
export async function sampleForFilter(
- spec: IngestionSpec,
+ spec: Partial<IngestionSpec>,
cacheRows: CacheRows,
): Promise<SampleResponse> {
const samplerType = getSpecType(spec);
@@ -490,7 +492,7 @@ export async function sampleForFilter(
const filter: any = deepGet(spec, 'spec.dataSchema.transformSpec.filter');
// Extra step to simulate auto detecting dimension with transforms
- const specialDimensionSpec: DimensionsSpec = {};
+ let specialDimensionSpec: DimensionsSpec = {};
if (transforms && transforms.length) {
const sampleSpecHack: SampleSpec = {
type: samplerType,
@@ -510,11 +512,15 @@ export async function sampleForFilter(
'filter-pre',
);
- specialDimensionSpec.dimensions = dedupe(
- headerFromSampleResponse({
- sampleResponse: sampleResponseHack,
- ignoreTimeColumn: true,
- }).concat(getDimensionNamesFromTransforms(transforms)),
+ specialDimensionSpec = deepSet(
+ specialDimensionSpec,
+ 'dimensions',
+ dedupe(
+ headerFromSampleResponse({
+ sampleResponse: sampleResponseHack,
+ ignoreTimeColumn: true,
+ }).concat(getDimensionNamesFromTransforms(transforms)),
+ ),
);
}
@@ -539,7 +545,7 @@ export async function sampleForFilter(
}
export async function sampleForSchema(
- spec: IngestionSpec,
+ spec: Partial<IngestionSpec>,
cacheRows: CacheRows,
): Promise<SampleResponse> {
const samplerType = getSpecType(spec);
diff --git a/web-console/src/views/load-data-view/filter-table/filter-table.tsx b/web-console/src/views/load-data-view/filter-table/filter-table.tsx
index 3b6905f..506c2b8 100644
--- a/web-console/src/views/load-data-view/filter-table/filter-table.tsx
+++ b/web-console/src/views/load-data-view/filter-table/filter-table.tsx
@@ -29,7 +29,7 @@ import './filter-table.scss';
export function filterTableSelectedColumnName(
sampleData: HeaderAndRows,
- selectedFilter: DruidFilter | undefined,
+ selectedFilter: Partial<DruidFilter> | undefined,
): string | undefined {
if (!selectedFilter) return;
const selectedFilterName = selectedFilter.dimension;
diff --git a/web-console/src/views/load-data-view/form-editor/__snapshots__/form-editor.spec.tsx.snap b/web-console/src/views/load-data-view/form-editor/__snapshots__/form-editor.spec.tsx.snap
new file mode 100644
index 0000000..141273b
--- /dev/null
+++ b/web-console/src/views/load-data-view/form-editor/__snapshots__/form-editor.spec.tsx.snap
@@ -0,0 +1,113 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`FormEditor matches snapshot 1`] = `
+<div
+ class="form-editor"
+>
+ <div
+ class="auto-form"
+ >
+ <div
+ class="bp3-form-group form-group-with-info"
+ >
+ <label
+ class="bp3-label"
+ >
+ Input format
+
+ <span
+ class="bp3-text-muted"
+ >
+ <span
+ class="bp3-popover2-target"
+ >
+ <span
+ class="bp3-icon bp3-icon-info-sign"
+ icon="info-sign"
+ >
+ <svg
+ data-icon="info-sign"
+ height="14"
+ viewBox="0 0 16 16"
+ width="14"
+ >
+ <desc>
+ info-sign
+ </desc>
+ <path
+ d="M8 0C3.58 0 0 3.58 0 8s3.58 8 8 8 8-3.58 8-8-3.58-8-8-8zM7 3h2v2H7V3zm3 10H6v-1h1V7H6V6h3v6h1v1z"
+ fill-rule="evenodd"
+ />
+ </svg>
+ </span>
+ </span>
+ </span>
+ </label>
+ <div
+ class="bp3-form-content"
+ >
+ <div
+ class="bp3-input-group bp3-intent-primary formatted-input-group suggestible-input"
+ >
+ <input
+ class="bp3-input"
+ placeholder=""
+ type="text"
+ value=""
+ />
+ </div>
+ </div>
+ </div>
+ </div>
+ <div
+ class="apply-cancel-buttons"
+ >
+ <button
+ class="bp3-button bp3-intent-danger delete"
+ type="button"
+ >
+ <span
+ class="bp3-icon bp3-icon-trash"
+ icon="trash"
+ >
+ <svg
+ data-icon="trash"
+ height="16"
+ viewBox="0 0 16 16"
+ width="16"
+ >
+ <desc>
+ trash
+ </desc>
+ <path
+ d="M14.49 3.99h-13c-.28 0-.5.22-.5.5s.22.5.5.5h.5v10c0 .55.45 1 1 1h10c.55 0 1-.45 1-1v-10h.5c.28 0 .5-.22.5-.5s-.22-.5-.5-.5zm-8.5 9c0 .55-.45 1-1 1s-1-.45-1-1v-6c0-.55.45-1 1-1s1 .45 1 1v6zm3 0c0 .55-.45 1-1 1s-1-.45-1-1v-6c0-.55.45-1 1-1s1 .45 1 1v6zm3 0c0 .55-.45 1-1 1s-1-.45-1-1v-6c0-.55.45-1 1-1s1 .45 1 1v6zm2-12h-4c0-.55-.45-1-1-1h-2c-.55 0-1 .45-1 1h-4c-.55 0-1 .45-1 1v1h14v-1c0-.55-.45-1-1-1z"
+ fill-rule="evenodd"
+ />
+ </svg>
+ </span>
+ </button>
+ <button
+ class="bp3-button"
+ type="button"
+ >
+ <span
+ class="bp3-button-text"
+ >
+ Cancel
+ </span>
+ </button>
+ <button
+ class="bp3-button bp3-disabled bp3-intent-primary"
+ disabled=""
+ tabindex="-1"
+ type="button"
+ >
+ <span
+ class="bp3-button-text"
+ >
+ Apply
+ </span>
+ </button>
+ </div>
+</div>
+`;
diff --git a/web-console/src/views/load-data-view/form-editor/form-editor.scss b/web-console/src/views/load-data-view/form-editor/form-editor.scss
new file mode 100644
index 0000000..99cc7e1
--- /dev/null
+++ b/web-console/src/views/load-data-view/form-editor/form-editor.scss
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@import '~@blueprintjs/core/src/common/colors';
+@import '../../../variables';
+
+.form-editor {
+ background: rgba($gray3, 0.15);
+ padding: 10px;
+ border-radius: $pt-border-radius;
+ margin-bottom: 15px;
+
+ .apply-cancel-buttons {
+ position: relative;
+ text-align: right;
+
+ .bp3-button {
+ margin-left: 15px;
+ }
+
+ .delete {
+ position: absolute;
+ left: 0;
+ margin-left: 0;
+ }
+ }
+}
diff --git a/web-console/src/views/load-data-view/form-editor/form-editor.spec.tsx b/web-console/src/views/load-data-view/form-editor/form-editor.spec.tsx
new file mode 100644
index 0000000..4311406
--- /dev/null
+++ b/web-console/src/views/load-data-view/form-editor/form-editor.spec.tsx
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { render } from '@testing-library/react';
+import React from 'react';
+
+import { FormEditor } from './form-editor';
+
+describe('FormEditor', () => {
+ it('matches snapshot', () => {
+ const applyCancelButtons = (
+ <FormEditor
+ fields={[
+ {
+ name: 'type',
+ label: 'Input format',
+ type: 'string',
+ required: true,
+ info: 'Info',
+ },
+ ]}
+ initValue={{}}
+ onClose={() => {}}
+ onDirty={() => {}}
+ onApply={() => {}}
+ showDelete
+ onDelete={() => {}}
+ />
+ );
+ const { container } = render(applyCancelButtons);
+ expect(container.firstChild).toMatchSnapshot();
+ });
+});
diff --git a/web-console/src/views/load-data-view/form-editor/form-editor.tsx b/web-console/src/views/load-data-view/form-editor/form-editor.tsx
new file mode 100644
index 0000000..51d0c5e
--- /dev/null
+++ b/web-console/src/views/load-data-view/form-editor/form-editor.tsx
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Button, Intent } from '@blueprintjs/core';
+import { IconNames } from '@blueprintjs/icons';
+import React, { useState } from 'react';
+
+import { AutoForm, Field } from '../../../components';
+
+import './form-editor.scss';
+
+export interface FormEditorProps<T> {
+ fields: Field<T>[];
+ initValue: T;
+ showCustom?: (thing: Partial<T>) => boolean;
+ onClose: () => void;
+ onDirty: () => void;
+ onApply: (thing: T) => void;
+ showDelete?: boolean;
+ disableDelete?: boolean;
+ onDelete?: () => void;
+ children?: any;
+}
+
+export function FormEditor<T>(props: FormEditorProps<T>) {
+ const {
+ fields,
+ initValue,
+ showCustom,
+ onDirty,
+ onApply,
+ onClose,
+ showDelete,
+ disableDelete,
+ onDelete,
+ children,
+ } = props;
+
+ const [currentValue, setCurrentValue] = useState<Partial<T>>(initValue);
+
+ return (
+ <div className="form-editor">
+ <AutoForm
+ fields={fields}
+ model={currentValue}
+ onChange={m => {
+ onDirty();
+ setCurrentValue(m);
+ }}
+ showCustom={showCustom}
+ />
+ {children}
+ <div className="apply-cancel-buttons">
+ {showDelete && onDelete && (
+ <Button
+ className="delete"
+ icon={IconNames.TRASH}
+ intent={Intent.DANGER}
+ disabled={disableDelete}
+ onClick={() => {
+ onDelete();
+ onClose();
+ }}
+ />
+ )}
+ <Button text="Cancel" onClick={onClose} />
+ <Button
+ text="Apply"
+ intent={Intent.PRIMARY}
+ disabled={currentValue === initValue || !AutoForm.isValidModel(currentValue, fields)}
+ onClick={() => {
+ onApply(currentValue as T);
+ onClose();
+ }}
+ />
+ </div>
+ </div>
+ );
+}
diff --git a/web-console/src/views/load-data-view/info-messages.tsx b/web-console/src/views/load-data-view/info-messages.tsx
index ee8a360..3377989 100644
--- a/web-console/src/views/load-data-view/info-messages.tsx
+++ b/web-console/src/views/load-data-view/info-messages.tsx
@@ -27,7 +27,7 @@ import { LearnMore } from './learn-more/learn-more';
export interface ConnectMessageProps {
inlineMode: boolean;
- spec: IngestionSpec;
+ spec: Partial<IngestionSpec>;
}
export const ConnectMessage = React.memo(function ConnectMessage(props: ConnectMessageProps) {
diff --git a/web-console/src/views/load-data-view/load-data-view.scss b/web-console/src/views/load-data-view/load-data-view.scss
index 8c4ec58..e3b01fe 100644
--- a/web-console/src/views/load-data-view/load-data-view.scss
+++ b/web-console/src/views/load-data-view/load-data-view.scss
@@ -304,16 +304,10 @@ $actual-icon-height: 400px;
}
.control-buttons {
- position: relative;
+ text-align: right;
.bp3-button {
- margin-right: 15px;
- }
-
- .right {
- position: absolute;
- right: 0;
- margin-right: 0;
+ margin-left: 15px;
}
}
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx
index b784faa..54949b9 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -166,6 +166,7 @@ import {
import { ExamplePicker } from './example-picker/example-picker';
import { FilterTable, filterTableSelectedColumnName } from './filter-table/filter-table';
+import { FormEditor } from './form-editor/form-editor';
import {
ConnectMessage,
FilterMessage,
@@ -208,14 +209,29 @@ function showRawLine(line: SampleEntry): string {
}
function showDruidLine(line: SampleEntry): string {
- if (!line.parsed) return 'No parse';
- return `Druid row: ${JSONBig.stringify(line.parsed)}`;
+ if (!line.input) return 'Invalid row';
+ return `Druid row: ${JSONBig.stringify(line.input)}`;
}
function showBlankLine(line: SampleEntry): string {
return line.parsed ? `[Row: ${JSONBig.stringify(line.parsed)}]` : '[Binary data]';
}
+function formatSampleEntries(sampleEntries: SampleEntry[], isDruidSource: boolean): string {
+ if (sampleEntries.length) {
+ if (isDruidSource) {
+ return sampleEntries.map(showDruidLine).join('\n');
+ }
+
+ return (sampleEntries.every(l => !l.parsed)
+ ? sampleEntries.map(showBlankLine)
+ : sampleEntries.map(showRawLine)
+ ).join('\n');
+ } else {
+ return 'No data returned from sampler';
+ }
+}
+
function getTimestampSpec(headerAndRows: HeaderAndRows | null): TimestampSpec {
if (!headerAndRows) return CONSTANT_TIMESTAMP_SPEC;
@@ -299,10 +315,15 @@ export interface LoadDataViewProps {
goToIngestion: (taskGroupId: string | undefined, supervisor?: string) => void;
}
+interface SelectedIndex<T> {
+ value: Partial<T>;
+ index: number;
+}
+
export interface LoadDataViewState {
step: Step;
- spec: IngestionSpec;
- specPreview: IngestionSpec;
+ spec: Partial<IngestionSpec>;
+ nextSpec?: Partial<IngestionSpec>;
cacheRows?: CacheRows;
// dialogs / modals
continueToSpec: boolean;
@@ -319,6 +340,7 @@ export interface LoadDataViewState {
sampleStrategy: SampleStrategy;
columnFilter: string;
specialColumnsOnly: boolean;
+ unsavedChange: boolean;
// for ioConfig
inputQueryState: QueryState<SampleResponseWithExtraInfo>;
@@ -327,25 +349,21 @@ export interface LoadDataViewState {
parserQueryState: QueryState<HeaderAndRows>;
// for flatten
- selectedFlattenFieldIndex: number;
- selectedFlattenField?: FlattenField;
+ selectedFlattenField?: SelectedIndex<FlattenField>;
// for timestamp
timestampQueryState: QueryState<{
headerAndRows: HeaderAndRows;
- spec: IngestionSpec;
+ spec: Partial<IngestionSpec>;
}>;
// for transform
transformQueryState: QueryState<HeaderAndRows>;
- selectedTransformIndex: number;
- selectedTransform?: Transform;
+ selectedTransform?: SelectedIndex<Transform>;
// for filter
filterQueryState: QueryState<HeaderAndRows>;
- selectedFilterIndex: number;
- selectedFilter?: DruidFilter;
- newFilterValue?: Record<string, any>;
+ selectedFilter?: SelectedIndex<DruidFilter>;
// for schema
schemaQueryState: QueryState<{
@@ -354,10 +372,8 @@ export interface LoadDataViewState {
metricsSpec: MetricSpec[] | undefined;
}>;
selectedAutoDimension?: string;
- selectedDimensionSpecIndex: number;
- selectedDimensionSpec?: DimensionSpec;
- selectedMetricSpecIndex: number;
- selectedMetricSpec?: MetricSpec;
+ selectedDimensionSpec?: SelectedIndex<DimensionSpec>;
+ selectedMetricSpec?: SelectedIndex<MetricSpec>;
// for final step
submitting: boolean;
@@ -372,7 +388,6 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
this.state = {
step: 'loading',
spec,
- specPreview: spec,
// dialogs / modals
showResetConfirm: false,
@@ -382,6 +397,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
sampleStrategy: 'start',
columnFilter: '',
specialColumnsOnly: false,
+ unsavedChange: false,
// for inputSource
inputQueryState: QueryState.INIT,
@@ -389,24 +405,17 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
// for parser
parserQueryState: QueryState.INIT,
- // for flatten
- selectedFlattenFieldIndex: -1,
-
// for timestamp
timestampQueryState: QueryState.INIT,
// for transform
transformQueryState: QueryState.INIT,
- selectedTransformIndex: -1,
// for filter
filterQueryState: QueryState.INIT,
- selectedFilterIndex: -1,
// for dimensions
schemaQueryState: QueryState.INIT,
- selectedDimensionSpecIndex: -1,
- selectedMetricSpecIndex: -1,
// for final step
submitting: false,
@@ -485,14 +494,33 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
}
}
+ private readonly handleDirty = () => {
+ this.setState({ unsavedChange: true });
+ };
+
private readonly updateStep = (newStep: Step) => {
- this.setState(state => ({ step: newStep, specPreview: state.spec }));
+ const { unsavedChange, nextSpec } = this.state;
+ if (unsavedChange || nextSpec) {
+ AppToaster.show({
+ message: `You have an unsaved change in this step.`,
+ intent: Intent.WARNING,
+ action: {
+ icon: IconNames.TRASH,
+ text: 'Discard change',
+ onClick: this.resetSelected,
+ },
+ });
+ return;
+ }
+
+ this.resetSelected();
+ this.setState({ step: newStep });
};
- private readonly updateSpec = (newSpec: IngestionSpec) => {
+ private readonly updateSpec = (newSpec: Partial<IngestionSpec>) => {
newSpec = normalizeSpec(newSpec);
newSpec = upgradeSpec(newSpec);
- const deltaState: Partial<LoadDataViewState> = { spec: newSpec, specPreview: newSpec };
+ const deltaState: Partial<LoadDataViewState> = { spec: newSpec };
if (!deepGet(newSpec, 'spec.ioConfig.type')) {
deltaState.cacheRows = undefined;
}
@@ -500,26 +528,37 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
localStorageSet(LocalStorageKeys.INGESTION_SPEC, JSONBig.stringify(newSpec));
};
- private readonly updateSpecPreview = (newSpecPreview: IngestionSpec) => {
- this.setState({ specPreview: newSpecPreview });
+ private readonly updateSpecPreview = (newSpecPreview: Partial<IngestionSpec>) => {
+ this.setState({ nextSpec: newSpecPreview });
};
private readonly applyPreviewSpec = () => {
- this.setState(({ spec, specPreview }) => {
- localStorageSet(LocalStorageKeys.INGESTION_SPEC, JSONBig.stringify(specPreview));
- return { spec: spec === specPreview ? { ...specPreview } : specPreview }; // If applying again, make a shallow copy to force a refresh
+ this.setState(({ spec, nextSpec }) => {
+ if (nextSpec) {
+ localStorageSet(LocalStorageKeys.INGESTION_SPEC, JSONBig.stringify(nextSpec));
+ }
+ return { spec: nextSpec ? nextSpec : { ...spec }, nextSpec: undefined }; // If applying again, make a shallow copy to force a refresh
});
};
- private readonly revertPreviewSpec = () => {
- this.setState(({ spec }) => ({ specPreview: spec }));
- };
-
- isPreviewSpecSame() {
- const { spec, specPreview } = this.state;
- return spec === specPreview;
+ private getEffectiveSpec() {
+ const { spec, nextSpec } = this.state;
+ return nextSpec || spec;
}
+ private readonly resetSelected = () => {
+ this.setState({
+ nextSpec: undefined,
+ selectedFlattenField: undefined,
+ selectedTransform: undefined,
+ selectedFilter: undefined,
+ selectedAutoDimension: undefined,
+ selectedDimensionSpec: undefined,
+ selectedMetricSpec: undefined,
+ unsavedChange: false,
+ });
+ };
+
componentDidUpdate(_prevProps: LoadDataViewProps, prevState: LoadDataViewState) {
const { spec, step } = this.state;
const { spec: prevSpec, step: prevStep } = prevState;
@@ -622,18 +661,18 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
}
renderApplyButtonBar(queryState: QueryState<unknown>, issue: string | undefined) {
- const previewSpecSame = this.isPreviewSpecSame();
+ const { nextSpec } = this.state;
const queryStateHasError = Boolean(queryState && queryState.error);
return (
<FormGroup className="control-buttons">
+ {nextSpec && <Button text="Cancel" onClick={this.resetSelected} />}
<Button
text="Apply"
- disabled={(previewSpecSame && !queryStateHasError) || Boolean(issue)}
+ disabled={(!nextSpec && !queryStateHasError) || Boolean(issue)}
intent={Intent.PRIMARY}
onClick={this.applyPreviewSpec}
/>
- {!previewSpecSame && <Button text="Cancel" onClick={this.revertPreviewSpec} />}
</FormGroup>
);
}
@@ -667,7 +706,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
renderNextBar(options: { nextStep?: Step; disabled?: boolean; onNextStep?: () => boolean }) {
const { disabled, onNextStep } = options;
- const { step } = this.state;
+ const { step, nextSpec, unsavedChange } = this.state;
const nextStep = options.nextStep || STEPS[STEPS.indexOf(step) + 1] || STEPS[0];
return (
@@ -676,9 +715,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
text={`Next: ${VIEW_TITLE[nextStep]}`}
rightIcon={IconNames.ARROW_RIGHT}
intent={Intent.PRIMARY}
- disabled={disabled || !this.isPreviewSpecSame()}
+ disabled={Boolean(disabled || nextSpec || unsavedChange)}
onClick={() => {
- if (disabled) return;
if (onNextStep && !onNextStep()) return;
this.updateStep(nextStep);
@@ -1042,7 +1080,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
private readonly handleResetSpec = () => {
this.setState({ showResetConfirm: false, continueToSpec: true });
- this.updateSpec({} as any);
+ this.updateSpec({});
this.updateStep('welcome');
};
@@ -1105,13 +1143,14 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
inputQueryState: new QueryState({ data: sampleResponse }),
};
if (isDruidSource(spec)) {
- deltaState.cacheRows = getCacheRowsFromSampleResponse(sampleResponse, true);
+ deltaState.cacheRows = getCacheRowsFromSampleResponse(sampleResponse);
}
this.setState(deltaState as LoadDataViewState);
}
renderConnectStep() {
- const { specPreview: spec, inputQueryState, sampleStrategy } = this.state;
+ const { inputQueryState, sampleStrategy } = this.state;
+ const spec = this.getEffectiveSpec();
const specType = getSpecType(spec);
const ioConfig: IoConfig = deepGet(spec, 'spec.ioConfig') || EMPTY_OBJECT;
const inlineMode = deepGet(spec, 'spec.ioConfig.inputSource.type') === 'inline';
@@ -1140,22 +1179,14 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
} else {
const data = inputQueryState.getSomeData();
const inputData = data ? data.data : undefined;
+
mainFill = (
<>
{inputData && (
<TextArea
className="raw-lines"
readOnly
- value={
- inputData.length
- ? (inputData.every(l => !l.parsed)
- ? inputData.map(showBlankLine)
- : druidSource
- ? inputData.map(showDruidLine)
- : inputData.map(showRawLine)
- ).join('\n')
- : 'No data returned from sampler'
- }
+ value={formatSampleEntries(inputData, druidSource)}
/>
)}
{inputQueryState.isLoading() && <Loader />}
@@ -1351,13 +1382,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
}
renderParserStep() {
- const {
- specPreview: spec,
- columnFilter,
- specialColumnsOnly,
- parserQueryState,
- selectedFlattenField,
- } = this.state;
+ const { columnFilter, specialColumnsOnly, parserQueryState, selectedFlattenField } = this.state;
+ const spec = this.getEffectiveSpec();
const inputFormat: InputFormat = deepGet(spec, 'spec.ioConfig.inputFormat') || EMPTY_OBJECT;
const flattenFields: FlattenField[] =
deepGet(spec, 'spec.ioConfig.inputFormat.flattenSpec.fields') || EMPTY_ARRAY;
@@ -1471,11 +1497,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
}
if (possibleTimestampSpec) {
- const newSpec: IngestionSpec = deepSet(
- spec,
- 'spec.dataSchema.timestampSpec',
- possibleTimestampSpec,
- );
+ const newSpec = deepSet(spec, 'spec.dataSchema.timestampSpec', possibleTimestampSpec);
this.updateSpec(newSpec);
}
@@ -1487,77 +1509,54 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
}
private readonly onFlattenFieldSelect = (field: FlattenField, index: number) => {
+ const { spec, unsavedChange } = this.state;
+ const inputFormat: InputFormat = deepGet(spec, 'spec.ioConfig.inputFormat') || EMPTY_OBJECT;
+ if (unsavedChange || !inputFormatCanFlatten(inputFormat)) return;
+
this.setState({
- selectedFlattenFieldIndex: index,
- selectedFlattenField: field,
+ selectedFlattenField: { value: field, index },
});
};
renderFlattenControls(): JSX.Element | undefined {
- const { spec, selectedFlattenField, selectedFlattenFieldIndex } = this.state;
- const inputFormat: InputFormat = deepGet(spec, 'spec.ioConfig.inputFormat') || EMPTY_OBJECT;
- if (!inputFormatCanFlatten(inputFormat)) return;
-
- const close = () => {
- this.setState({
- selectedFlattenFieldIndex: -1,
- selectedFlattenField: undefined,
- });
- };
+ const { spec, nextSpec, selectedFlattenField } = this.state;
if (selectedFlattenField) {
return (
- <div className="edit-controls">
- <AutoForm
- fields={FLATTEN_FIELD_FIELDS}
- model={selectedFlattenField}
- onChange={f => this.setState({ selectedFlattenField: f })}
- />
- <div className="control-buttons">
- <Button
- text="Apply"
- intent={Intent.PRIMARY}
- onClick={() => {
- this.updateSpec(
- deepSet(
- spec,
- `spec.ioConfig.inputFormat.flattenSpec.fields.${selectedFlattenFieldIndex}`,
- selectedFlattenField,
- ),
- );
- close();
- }}
- />
- <Button text="Cancel" onClick={close} />
- {selectedFlattenFieldIndex !== -1 && (
- <Button
- className="right"
- icon={IconNames.TRASH}
- intent={Intent.DANGER}
- onClick={() => {
- this.updateSpec(
- deepDelete(
- spec,
- `spec.ioConfig.inputFormat.flattenSpec.fields.${selectedFlattenFieldIndex}`,
- ),
- );
- close();
- }}
- />
- )}
- </div>
- </div>
+ <FormEditor
+ fields={FLATTEN_FIELD_FIELDS}
+ initValue={selectedFlattenField.value}
+ onClose={this.resetSelected}
+ onDirty={this.handleDirty}
+ onApply={flattenField =>
+ this.updateSpec(
+ deepSet(
+ spec,
+ `spec.ioConfig.inputFormat.flattenSpec.fields.${selectedFlattenField.index}`,
+ flattenField,
+ ),
+ )
+ }
+ showDelete={selectedFlattenField.index !== -1}
+ onDelete={() =>
+ this.updateSpec(
+ deepDelete(
+ spec,
+ `spec.ioConfig.inputFormat.flattenSpec.fields.${selectedFlattenField.index}`,
+ ),
+ )
+ }
+ />
);
} else {
return (
<FormGroup>
<Button
text="Add column flattening"
- disabled={!this.isPreviewSpecSame()}
+ disabled={Boolean(nextSpec)}
onClick={() => {
this.setState({
- selectedFlattenField: { type: 'path', name: '', expr: '' },
- selectedFlattenFieldIndex: -1,
+ selectedFlattenField: { value: { type: 'path' }, index: -1 },
});
}}
/>
@@ -1623,7 +1622,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
}
renderTimestampStep() {
- const { specPreview: spec, columnFilter, specialColumnsOnly, timestampQueryState } = this.state;
+ const { columnFilter, specialColumnsOnly, timestampQueryState } = this.state;
+ const spec = this.getEffectiveSpec();
const timestampSchema = getTimestampSchema(spec);
const timestampSpec: TimestampSpec =
deepGet(spec, 'spec.dataSchema.timestampSpec') || EMPTY_OBJECT;
@@ -1760,8 +1760,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
}
private readonly onTimestampColumnSelect = (newTimestampSpec: TimestampSpec) => {
- const { specPreview } = this.state;
- this.updateSpecPreview(deepSet(specPreview, 'spec.dataSchema.timestampSpec', newTimestampSpec));
+ const spec = this.getEffectiveSpec();
+ this.updateSpecPreview(deepSet(spec, 'spec.dataSchema.timestampSpec', newTimestampSpec));
};
// ==================================================================
@@ -1818,7 +1818,6 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
specialColumnsOnly,
transformQueryState,
selectedTransform,
- // selectedTransformIndex,
} = this.state;
const transforms: Transform[] =
deepGet(spec, 'spec.dataSchema.transformSpec.transforms') || EMPTY_ARRAY;
@@ -1849,7 +1848,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
columnFilter={columnFilter}
transformedColumnsOnly={specialColumnsOnly}
transforms={transforms}
- selectedColumnName={transformTableSelectedColumnName(data, selectedTransform)}
+ selectedColumnName={transformTableSelectedColumnName(data, selectedTransform?.value)}
onTransformSelect={this.onTransformSelect}
/>
)}
@@ -1874,8 +1873,10 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
intent={Intent.PRIMARY}
onClick={() => {
this.setState({
- selectedTransformIndex: transforms.length - 1,
- selectedTransform: transforms[transforms.length - 1],
+ selectedTransform: {
+ value: transforms[transforms.length - 1],
+ index: transforms.length - 1,
+ },
});
}}
/>
@@ -1907,65 +1908,44 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
);
}
- private readonly onTransformSelect = (transform: Transform, index: number) => {
+ private readonly onTransformSelect = (transform: Partial<Transform>, index: number) => {
+ const { unsavedChange } = this.state;
+ if (unsavedChange) return;
+
this.setState({
- selectedTransformIndex: index,
- selectedTransform: transform,
+ selectedTransform: { value: transform, index },
});
};
renderTransformControls() {
- const { spec, selectedTransform, selectedTransformIndex } = this.state;
-
- const close = () => {
- this.setState({
- selectedTransformIndex: -1,
- selectedTransform: undefined,
- });
- };
+ const { spec, selectedTransform } = this.state;
if (selectedTransform) {
return (
- <div className="edit-controls">
- <AutoForm
- fields={TRANSFORM_FIELDS}
- model={selectedTransform}
- onChange={selectedTransform => this.setState({ selectedTransform })}
- />
- <div className="control-buttons">
- <Button
- text="Apply"
- intent={Intent.PRIMARY}
- onClick={() => {
- this.updateSpec(
- deepSet(
- spec,
- `spec.dataSchema.transformSpec.transforms.${selectedTransformIndex}`,
- selectedTransform,
- ),
- );
- close();
- }}
- />
- <Button text="Cancel" onClick={close} />
- {selectedTransformIndex !== -1 && (
- <Button
- className="right"
- icon={IconNames.TRASH}
- intent={Intent.DANGER}
- onClick={() => {
- this.updateSpec(
- deepDelete(
- spec,
- `spec.dataSchema.transformSpec.transforms.${selectedTransformIndex}`,
- ),
- );
- close();
- }}
- />
- )}
- </div>
- </div>
+ <FormEditor
+ fields={TRANSFORM_FIELDS}
+ initValue={selectedTransform.value}
+ onClose={this.resetSelected}
+ onDirty={this.handleDirty}
+ onApply={transform =>
+ this.updateSpec(
+ deepSet(
+ spec,
+ `spec.dataSchema.transformSpec.transforms.${selectedTransform.index}`,
+ transform,
+ ),
+ )
+ }
+ showDelete={selectedTransform.index !== -1}
+ onDelete={() =>
+ this.updateSpec(
+ deepDelete(
+ spec,
+ `spec.dataSchema.transformSpec.transforms.${selectedTransform.index}`,
+ ),
+ )
+ }
+ />
);
} else {
return (
@@ -1973,10 +1953,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
<Button
text="Add column transform"
onClick={() => {
- this.setState({
- selectedTransformIndex: -1,
- selectedTransform: { type: 'expression', name: '', expression: '' },
- });
+ this.onTransformSelect({ type: 'expression' }, -1);
}}
/>
</FormGroup>
@@ -2060,7 +2037,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
});
renderFilterStep() {
- const { spec, specPreview, columnFilter, filterQueryState, selectedFilter } = this.state;
+ const { columnFilter, filterQueryState, selectedFilter } = this.state;
+ const spec = this.getEffectiveSpec();
const dimensionFilters = this.getMemoizedDimensionFiltersFromSpec(spec);
let mainFill: JSX.Element | string;
@@ -2082,7 +2060,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
sampleData={data}
columnFilter={columnFilter}
dimensionFilters={dimensionFilters}
- selectedFilterName={filterTableSelectedColumnName(data, selectedFilter)}
+ selectedFilterName={filterTableSelectedColumnName(data, selectedFilter?.value)}
onFilterSelect={this.onFilterSelect}
/>
)}
@@ -2101,19 +2079,14 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
<FilterMessage />
{!selectedFilter && (
<>
- <AutoForm
- fields={FILTERS_FIELDS}
- model={specPreview}
- onChange={this.updateSpecPreview}
- />
+ <AutoForm fields={FILTERS_FIELDS} model={spec} onChange={this.updateSpecPreview} />
{this.renderApplyButtonBar(filterQueryState, undefined)}
<FormGroup>
<Button
text="Add column filter"
onClick={() => {
this.setState({
- selectedFilter: { type: 'selector', dimension: '', value: '' },
- selectedFilterIndex: -1,
+ selectedFilter: { value: { type: 'selector' }, index: -1 },
});
}}
/>
@@ -2147,63 +2120,37 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
private readonly onFilterSelect = (filter: DruidFilter, index: number) => {
this.setState({
- selectedFilterIndex: index,
- selectedFilter: filter,
+ selectedFilter: { value: filter, index },
});
};
renderColumnFilterControls() {
- const { spec, selectedFilter, selectedFilterIndex } = this.state;
+ const { spec, selectedFilter } = this.state;
if (!selectedFilter) return;
- const close = () => {
- this.setState({
- selectedFilterIndex: -1,
- selectedFilter: undefined,
- });
- };
-
return (
- <div className="edit-controls">
- <AutoForm
- fields={FILTER_FIELDS}
- model={selectedFilter}
- onChange={f => this.setState({ selectedFilter: f })}
- showCustom={f => !KNOWN_FILTER_TYPES.includes(f.type)}
- />
- <div className="control-buttons">
- <Button
- text="Apply"
- intent={Intent.PRIMARY}
- onClick={() => {
- const curFilter = splitFilter(deepGet(spec, 'spec.dataSchema.transformSpec.filter'));
- const newFilter = joinFilter(
- deepSet(curFilter, `dimensionFilters.${selectedFilterIndex}`, selectedFilter),
- );
- this.updateSpec(deepSet(spec, 'spec.dataSchema.transformSpec.filter', newFilter));
- close();
- }}
- />
- <Button text="Cancel" onClick={close} />
- {selectedFilterIndex !== -1 && (
- <Button
- className="right"
- icon={IconNames.TRASH}
- intent={Intent.DANGER}
- onClick={() => {
- const curFilter = splitFilter(
- deepGet(spec, 'spec.dataSchema.transformSpec.filter'),
- );
- const newFilter = joinFilter(
- deepDelete(curFilter, `dimensionFilters.${selectedFilterIndex}`),
- );
- this.updateSpec(deepSet(spec, 'spec.dataSchema.transformSpec.filter', newFilter));
- close();
- }}
- />
- )}
- </div>
- </div>
+ <FormEditor
+ fields={FILTER_FIELDS}
+ initValue={selectedFilter.value}
+ showCustom={f => !KNOWN_FILTER_TYPES.includes(f.type || '')}
+ onClose={this.resetSelected}
+ onDirty={this.handleDirty}
+ onApply={filter => {
+ const curFilter = splitFilter(deepGet(spec, 'spec.dataSchema.transformSpec.filter'));
+ const newFilter = joinFilter(
+ deepSet(curFilter, `dimensionFilters.${selectedFilter.index}`, filter),
+ );
+ this.updateSpec(deepSet(spec, 'spec.dataSchema.transformSpec.filter', newFilter));
+ }}
+ showDelete={selectedFilter.index !== -1}
+ onDelete={() => {
+ const curFilter = splitFilter(deepGet(spec, 'spec.dataSchema.transformSpec.filter'));
+ const newFilter = joinFilter(
+ deepDelete(curFilter, `dimensionFilters.${selectedFilter.index}`),
+ );
+ this.updateSpec(deepSet(spec, 'spec.dataSchema.transformSpec.filter', newFilter));
+ }}
+ />
);
}
@@ -2263,15 +2210,13 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
renderSchemaStep() {
const {
- specPreview: spec,
columnFilter,
schemaQueryState,
selectedAutoDimension,
selectedDimensionSpec,
- selectedDimensionSpecIndex,
selectedMetricSpec,
- selectedMetricSpecIndex,
} = this.state;
+ const spec = this.getEffectiveSpec();
const rollup = Boolean(deepGet(spec, 'spec.dataSchema.granularitySpec.rollup'));
const somethingSelected = Boolean(
selectedAutoDimension || selectedDimensionSpec || selectedMetricSpec,
@@ -2297,8 +2242,8 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
sampleBundle={data}
columnFilter={columnFilter}
selectedAutoDimension={selectedAutoDimension}
- selectedDimensionSpecIndex={selectedDimensionSpecIndex}
- selectedMetricSpecIndex={selectedMetricSpecIndex}
+ selectedDimensionSpecIndex={selectedDimensionSpec ? selectedDimensionSpec.index : -1}
+ selectedMetricSpecIndex={selectedMetricSpec ? selectedMetricSpec.index : -1}
onAutoDimensionSelect={this.onAutoDimensionSelect}
onDimensionSelect={this.onDimensionSelect}
onMetricSelect={this.onMetricSelect}
@@ -2441,10 +2386,11 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
disabled={dimensionMode !== 'specific'}
onClick={() => {
this.setState({
- selectedDimensionSpecIndex: -1,
selectedDimensionSpec: {
- name: 'new_dimension',
- type: 'string',
+ value: {
+ type: 'string',
+ },
+ index: -1,
},
});
}}
@@ -2455,11 +2401,11 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
text="Add metric"
onClick={() => {
this.setState({
- selectedMetricSpecIndex: -1,
selectedMetricSpec: {
- name: 'sum_blah',
- type: 'doubleSum',
- fieldName: '',
+ value: {
+ type: 'doubleSum',
+ },
+ index: -1,
},
});
}}
@@ -2544,35 +2490,23 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
this.setState({
selectedAutoDimension,
selectedDimensionSpec: undefined,
- selectedDimensionSpecIndex: -1,
selectedMetricSpec: undefined,
- selectedMetricSpecIndex: -1,
});
};
- private readonly onDimensionSelect = (
- selectedDimensionSpec: DimensionSpec | undefined,
- selectedDimensionSpecIndex: number,
- ) => {
+ private readonly onDimensionSelect = (dimensionSpec: DimensionSpec, index: number) => {
this.setState({
selectedAutoDimension: undefined,
- selectedDimensionSpec,
- selectedDimensionSpecIndex,
+ selectedDimensionSpec: { value: dimensionSpec, index },
selectedMetricSpec: undefined,
- selectedMetricSpecIndex: -1,
});
};
- private readonly onMetricSelect = (
- selectedMetricSpec: MetricSpec | undefined,
- selectedMetricSpecIndex: number,
- ) => {
+ private readonly onMetricSelect = (metricSpec: MetricSpec, index: number) => {
this.setState({
selectedAutoDimension: undefined,
selectedDimensionSpec: undefined,
- selectedDimensionSpecIndex: -1,
- selectedMetricSpec,
- selectedMetricSpecIndex,
+ selectedMetricSpec: { value: metricSpec, index },
});
};
@@ -2646,12 +2580,6 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
const { spec, selectedAutoDimension } = this.state;
if (!selectedAutoDimension) return;
- const close = () => {
- this.setState({
- selectedAutoDimension: undefined,
- });
- };
-
return (
<div className="edit-controls">
<FormGroup label="Name">
@@ -2670,40 +2598,37 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
selectedAutoDimension,
),
);
- close();
+ this.resetSelected();
}}
/>
</FormGroup>
<FormGroup>
- <Button text="Close" onClick={close} />
+ <Button text="Close" onClick={this.resetSelected} />
</FormGroup>
</div>
);
}
renderDimensionSpecControls() {
- const { spec, selectedDimensionSpec, selectedDimensionSpecIndex } = this.state;
+ const { spec, selectedDimensionSpec } = this.state;
if (!selectedDimensionSpec) return;
const dimensionMode = getDimensionMode(spec);
- const close = () => {
- this.setState({
- selectedDimensionSpecIndex: -1,
- selectedDimensionSpec: undefined,
- });
- };
-
const dimensions = deepGet(spec, `spec.dataSchema.dimensionsSpec.dimensions`) || EMPTY_ARRAY;
const moveTo = (newIndex: number) => {
- const newDimension = moveElement(dimensions, selectedDimensionSpecIndex, newIndex);
+ const newDimension = moveElement(dimensions, selectedDimensionSpec.index, newIndex);
const newSpec = deepSet(spec, `spec.dataSchema.dimensionsSpec.dimensions`, newDimension);
this.updateSpec(newSpec);
- close();
+ this.resetSelected();
};
const reorderDimensionMenu = (
- <ReorderMenu things={dimensions} selectedIndex={selectedDimensionSpecIndex} moveTo={moveTo} />
+ <ReorderMenu
+ things={dimensions}
+ selectedIndex={selectedDimensionSpec.index}
+ moveTo={moveTo}
+ />
);
const convertToMetric = (type: string, prefix: string) => {
@@ -2711,18 +2636,18 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
dimensionMode === 'specific'
? deepDelete(
spec,
- `spec.dataSchema.dimensionsSpec.dimensions.${selectedDimensionSpecIndex}`,
+ `spec.dataSchema.dimensionsSpec.dimensions.${selectedDimensionSpec.index}`,
)
: spec;
const specWithMetric = deepSet(specWithoutDimension, `spec.dataSchema.metricsSpec.[append]`, {
- name: `${prefix}_${selectedDimensionSpec.name}`,
+ name: `${prefix}_${selectedDimensionSpec.value.name}`,
type,
- fieldName: selectedDimensionSpec.name,
+ fieldName: selectedDimensionSpec.value.name,
});
this.updateSpec(specWithMetric);
- close();
+ this.resetSelected();
};
const convertToMetricMenu = (
@@ -2755,13 +2680,32 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
);
return (
- <div className="edit-controls">
- <AutoForm
- fields={DIMENSION_SPEC_FIELDS}
- model={selectedDimensionSpec}
- onChange={selectedDimensionSpec => this.setState({ selectedDimensionSpec })}
- />
- {selectedDimensionSpecIndex !== -1 && (
+ <FormEditor
+ fields={DIMENSION_SPEC_FIELDS}
+ initValue={selectedDimensionSpec.value}
+ onClose={this.resetSelected}
+ onDirty={this.handleDirty}
+ onApply={dimensionSpec =>
+ this.updateSpec(
+ deepSet(
+ spec,
+ `spec.dataSchema.dimensionsSpec.dimensions.${selectedDimensionSpec.index}`,
+ dimensionSpec,
+ ),
+ )
+ }
+ showDelete={selectedDimensionSpec.index !== -1}
+ disableDelete={dimensions.length <= 1}
+ onDelete={() =>
+ this.updateSpec(
+ deepDelete(
+ spec,
+ `spec.dataSchema.dimensionsSpec.dimensions.${selectedDimensionSpec.index}`,
+ ),
+ )
+ }
+ >
+ {selectedDimensionSpec.index !== -1 && (
<FormGroup>
<Popover2 content={reorderDimensionMenu}>
<Button
@@ -2772,7 +2716,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
</Popover2>
</FormGroup>
)}
- {selectedDimensionSpecIndex !== -1 && deepGet(spec, 'spec.dataSchema.metricsSpec') && (
+ {selectedDimensionSpec.index !== -1 && deepGet(spec, 'spec.dataSchema.metricsSpec') && (
<FormGroup>
<Popover2 content={convertToMetricMenu}>
<Button
@@ -2784,62 +2728,21 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
</Popover2>
</FormGroup>
)}
- <div className="control-buttons">
- <Button
- text="Apply"
- intent={Intent.PRIMARY}
- onClick={() => {
- this.updateSpec(
- deepSet(
- spec,
- `spec.dataSchema.dimensionsSpec.dimensions.${selectedDimensionSpecIndex}`,
- selectedDimensionSpec,
- ),
- );
- close();
- }}
- />
- <Button text="Cancel" onClick={close} />
- {selectedDimensionSpecIndex !== -1 && (
- <Button
- className="right"
- icon={IconNames.TRASH}
- intent={Intent.DANGER}
- disabled={dimensions.length <= 1}
- onClick={() => {
- if (dimensions.length <= 1) return; // Guard against removing the last dimension
-
- this.updateSpec(
- deepDelete(
- spec,
- `spec.dataSchema.dimensionsSpec.dimensions.${selectedDimensionSpecIndex}`,
- ),
- );
- close();
- }}
- />
- )}
- </div>
- </div>
+ </FormEditor>
);
}
renderMetricSpecControls() {
- const { spec, selectedMetricSpec, selectedMetricSpecIndex } = this.state;
+ const { spec, selectedMetricSpec } = this.state;
if (!selectedMetricSpec) return;
const dimensionMode = getDimensionMode(spec);
-
- const close = () => {
- this.setState({
- selectedMetricSpecIndex: -1,
- selectedMetricSpec: undefined,
- });
- };
+ const selectedMetricSpecFieldName = selectedMetricSpec.value.fieldName;
const convertToDimension = (type: string) => {
+ if (!selectedMetricSpecFieldName) return;
const specWithoutMetric = deepDelete(
spec,
- `spec.dataSchema.metricsSpec.${selectedMetricSpecIndex}`,
+ `spec.dataSchema.metricsSpec.${selectedMetricSpec.index}`,
);
const specWithDimension = deepSet(
@@ -2847,12 +2750,12 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
`spec.dataSchema.dimensionsSpec.dimensions.[append]`,
{
type,
- name: selectedMetricSpec.fieldName,
+ name: selectedMetricSpecFieldName,
},
);
this.updateSpec(specWithDimension);
- close();
+ this.resetSelected();
};
const convertToDimensionMenu = (
@@ -2865,54 +2768,37 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
);
return (
- <div className="edit-controls">
- <AutoForm
- fields={METRIC_SPEC_FIELDS}
- model={selectedMetricSpec}
- onChange={selectedMetricSpec => this.setState({ selectedMetricSpec })}
- />
- {selectedMetricSpecIndex !== -1 && dimensionMode === 'specific' && (
- <FormGroup>
- <Popover2 content={convertToDimensionMenu}>
- <Button
- icon={IconNames.EXCHANGE}
- text="Convert to dimension"
- rightIcon={IconNames.CARET_DOWN}
- />
- </Popover2>
- </FormGroup>
- )}
- <div className="control-buttons">
- <Button
- text="Apply"
- intent={Intent.PRIMARY}
- onClick={() => {
- this.updateSpec(
- deepSet(
- spec,
- `spec.dataSchema.metricsSpec.${selectedMetricSpecIndex}`,
- selectedMetricSpec,
- ),
- );
- close();
- }}
- />
- <Button text="Cancel" onClick={close} />
- {selectedMetricSpecIndex !== -1 && (
- <Button
- className="right"
- icon={IconNames.TRASH}
- intent={Intent.DANGER}
- onClick={() => {
- this.updateSpec(
- deepDelete(spec, `spec.dataSchema.metricsSpec.${selectedMetricSpecIndex}`),
- );
- close();
- }}
- />
+ <FormEditor
+ fields={METRIC_SPEC_FIELDS}
+ initValue={selectedMetricSpec.value}
+ onClose={this.resetSelected}
+ onDirty={this.handleDirty}
+ onApply={metricSpec =>
+ this.updateSpec(
+ deepSet(spec, `spec.dataSchema.metricsSpec.${selectedMetricSpec.index}`, metricSpec),
+ )
+ }
+ showDelete={selectedMetricSpec.index !== -1}
+ onDelete={() =>
+ this.updateSpec(
+ deepDelete(spec, `spec.dataSchema.metricsSpec.${selectedMetricSpec.index}`),
+ )
+ }
+ >
+ {selectedMetricSpec.index !== -1 &&
+ dimensionMode === 'specific' &&
+ selectedMetricSpecFieldName && (
+ <FormGroup>
+ <Popover2 content={convertToDimensionMenu}>
+ <Button
+ icon={IconNames.EXCHANGE}
+ text="Convert to dimension"
+ rightIcon={IconNames.CARET_DOWN}
+ />
+ </Popover2>
+ </FormGroup>
)}
- </div>
- </div>
+ </FormEditor>
);
}
diff --git a/web-console/src/views/load-data-view/parse-time-table/parse-time-table.tsx b/web-console/src/views/load-data-view/parse-time-table/parse-time-table.tsx
index 608952e..e592d1e 100644
--- a/web-console/src/views/load-data-view/parse-time-table/parse-time-table.tsx
+++ b/web-console/src/views/load-data-view/parse-time-table/parse-time-table.tsx
@@ -47,7 +47,7 @@ export function parseTimeTableSelectedColumnName(
export interface ParseTimeTableProps {
sampleBundle: {
headerAndRows: HeaderAndRows;
- spec: IngestionSpec;
+ spec: Partial<IngestionSpec>;
};
columnFilter: string;
possibleTimestampColumnsOnly: boolean;
diff --git a/web-console/src/views/load-data-view/schema-table/schema-table.tsx b/web-console/src/views/load-data-view/schema-table/schema-table.tsx
index a657b93..e5226c0 100644
--- a/web-console/src/views/load-data-view/schema-table/schema-table.tsx
+++ b/web-console/src/views/load-data-view/schema-table/schema-table.tsx
@@ -45,14 +45,8 @@ export interface SchemaTableProps {
selectedDimensionSpecIndex: number;
selectedMetricSpecIndex: number;
onAutoDimensionSelect: (dimensionName: string) => void;
- onDimensionSelect: (
- selectedDimensionSpec: DimensionSpec | undefined,
- selectedDimensionSpecIndex: number,
- ) => void;
- onMetricSelect: (
- selectedMetricSpec: MetricSpec | undefined,
- selectedMetricSpecIndex: number,
- ) => void;
+ onDimensionSelect: (dimensionSpec: DimensionSpec, index: number) => void;
+ onMetricSelect: (metricSpec: MetricSpec, index: number) => void;
}
export const SchemaTable = React.memo(function SchemaTable(props: SchemaTableProps) {
diff --git a/web-console/src/views/load-data-view/transform-table/transform-table.tsx b/web-console/src/views/load-data-view/transform-table/transform-table.tsx
index 4feadb2..c2bfc42 100644
--- a/web-console/src/views/load-data-view/transform-table/transform-table.tsx
+++ b/web-console/src/views/load-data-view/transform-table/transform-table.tsx
@@ -30,11 +30,11 @@ import './transform-table.scss';
export function transformTableSelectedColumnName(
sampleData: HeaderAndRows,
- selectedTransform: Transform | undefined,
+ selectedTransform: Partial<Transform> | undefined,
): string | undefined {
if (!selectedTransform) return;
const selectedTransformName = selectedTransform.name;
- if (!sampleData.header.includes(selectedTransformName)) return;
+ if (selectedTransformName && !sampleData.header.includes(selectedTransformName)) return;
return selectedTransformName;
}
diff --git a/web-console/src/views/lookups-view/lookups-view.tsx b/web-console/src/views/lookups-view/lookups-view.tsx
index c6e1588..309da34 100644
--- a/web-console/src/views/lookups-view/lookups-view.tsx
+++ b/web-console/src/views/lookups-view/lookups-view.tsx
@@ -69,7 +69,7 @@ export interface LookupEditInfo {
name: string;
tier: string;
version: string;
- spec: LookupSpec;
+ spec: Partial<LookupSpec>;
}
export interface LookupsViewProps {}
diff --git a/web-console/src/views/query-view/column-tree/column-tree.tsx b/web-console/src/views/query-view/column-tree/column-tree.tsx
index 6ae9bd9..f246bd2 100644
--- a/web-console/src/views/query-view/column-tree/column-tree.tsx
+++ b/web-console/src/views/query-view/column-tree/column-tree.tsx
@@ -42,6 +42,15 @@ import './column-tree.scss';
const LAST_DAY = SqlExpression.parse(`__time >= CURRENT_TIMESTAMP - INTERVAL '1' DAY`);
const COUNT_STAR = SqlFunction.COUNT_STAR.as('Count');
+function getCountExpression(columnNames: string[]): SqlExpression {
+ for (const columnName of columnNames) {
+ if (columnName === 'count' || columnName === '__count') {
+ return SqlFunction.simple('SUM', [SqlRef.column(columnName)]).as('Count');
+ }
+ }
+ return COUNT_STAR;
+}
+
const STRING_QUERY = SqlQuery.parse(`SELECT
?
FROM ?
@@ -169,30 +178,64 @@ export class ColumnTree extends React.PureComponent<ColumnTreeProps, ColumnTreeS
const parsedQuery = props.getParsedQuery();
const tableRef = SqlTableRef.create(tableName);
const prettyTableRef = prettyPrintSql(tableRef);
+ const countExpression = getCountExpression(
+ metadata.map(child => child.COLUMN_NAME),
+ );
+
+ const getQueryOnTable = () => {
+ return SqlQuery.create(
+ SqlTableRef.create(
+ tableName,
+ schemaName === 'druid' ? undefined : schemaName,
+ ),
+ );
+ };
+
+ const getWhere = (defaultToAllTime = false) => {
+ if (parsedQuery && parsedQuery.getFirstTableName() === tableName) {
+ return parsedQuery.getWhereExpression();
+ } else if (schemaName === 'druid') {
+ return defaultToAllTime ? undefined : LAST_DAY;
+ } else {
+ return;
+ }
+ };
+
return (
<Menu>
<MenuItem
icon={IconNames.FULLSCREEN}
- text={`SELECT ... FROM ${tableName}`}
+ text={`SELECT ...columns... FROM ${tableName}`}
onClick={() => {
- const tableRef = SqlTableRef.create(
- tableName,
- schemaName === 'druid' ? undefined : schemaName,
- );
-
- let where: SqlExpression | undefined;
- if (parsedQuery && parsedQuery.getFirstTableName() === tableName) {
- where = parsedQuery.getWhereExpression();
- } else if (schemaName === 'druid') {
- where = LAST_DAY;
- }
-
onQueryChange(
- SqlQuery.create(tableRef)
+ getQueryOnTable()
.changeSelectExpressions(
metadata.map(child => SqlRef.column(child.COLUMN_NAME)),
)
- .changeWhereExpression(where),
+ .changeWhereExpression(getWhere()),
+ true,
+ );
+ }}
+ />
+ <MenuItem
+ icon={IconNames.FULLSCREEN}
+ text={`SELECT * FROM ${tableName}`}
+ onClick={() => {
+ onQueryChange(
+ getQueryOnTable().changeWhereExpression(getWhere()),
+ true,
+ );
+ }}
+ />
+ <MenuItem
+ icon={IconNames.FULLSCREEN}
+ text={`SELECT ${countExpression} FROM ${tableName}`}
+ onClick={() => {
+ onQueryChange(
+ getQueryOnTable()
+ .changeSelect(0, countExpression)
+ .changeGroupByExpressions([])
+ .changeWhereExpression(getWhere(true)),
true,
);
}}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org