You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by fj...@apache.org on 2019/06/19 19:48:37 UTC
[incubator-druid] branch master updated: Web console: Data loader
respects parse spec columns for data preview (#7922)
This is an automated email from the ASF dual-hosted git repository.
fjy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 28eaa62 Web console: Data loader respects parse spec columns for data preview (#7922)
28eaa62 is described below
commit 28eaa620a95eacefd722e20de5df9282271394a3
Author: Vadim Ogievetsky <va...@gmail.com>
AuthorDate: Wed Jun 19 12:48:31 2019 -0700
Web console: Data loader respects parse spec columns for data preview (#7922)
* small fixes in the data loader
* respect columns
* fix test
---
web-console/package-lock.json | 5 +
web-console/package.json | 1 +
web-console/src/utils/general.spec.ts | 31 ++++
web-console/src/utils/general.tsx | 25 +++-
web-console/src/utils/sampler.ts | 23 +--
.../src/views/load-data-view/load-data-view.scss | 10 +-
.../src/views/load-data-view/load-data-view.tsx | 157 +++++++++++----------
.../load-data-view/schema-table/schema-table.tsx | 4 +-
8 files changed, 158 insertions(+), 98 deletions(-)
diff --git a/web-console/package-lock.json b/web-console/package-lock.json
index d422113..a58881a 100644
--- a/web-console/package-lock.json
+++ b/web-console/package-lock.json
@@ -5175,6 +5175,11 @@
"integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
"dev": true
},
+ "has-own-prop": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/has-own-prop/-/has-own-prop-2.0.0.tgz",
+ "integrity": "sha512-Pq0h+hvsVm6dDEa8x82GnLSYHOzNDt7f0ddFa3FqcQlgzEiptPqL+XrOJNavjOzSYiYWIrgeVYYgGlLmnxwilQ=="
+ },
"has-symbols": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz",
diff --git a/web-console/package.json b/web-console/package.json
index f55ebe9..46958da 100644
--- a/web-console/package.json
+++ b/web-console/package.json
@@ -53,6 +53,7 @@
"es6-shim": "^0.35.5",
"es7-shim": "^6.0.0",
"file-saver": "^2.0.2",
+ "has-own-prop": "^2.0.0",
"hjson": "^3.1.2",
"lodash.debounce": "^4.0.8",
"memoize-one": "^5.0.4",
diff --git a/web-console/src/utils/general.spec.ts b/web-console/src/utils/general.spec.ts
new file mode 100644
index 0000000..46b2355
--- /dev/null
+++ b/web-console/src/utils/general.spec.ts
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { alphanumericCompare, sortWithPrefixSuffix } from './general';
+
+describe('general', () => {
+ describe('sortWithPrefixSuffix', () => {
+ it('works in simple case', () => {
+ expect(sortWithPrefixSuffix('abcdefgh'.split('').reverse(), 'gef'.split(''), 'ba'.split(''), alphanumericCompare).join('')).toEqual('gefcdhba');
+ });
+
+ it('dedupes', () => {
+ expect(sortWithPrefixSuffix('abcdefgh'.split('').reverse(), 'gefgef'.split(''), 'baba'.split(''), alphanumericCompare).join('')).toEqual('gefcdhba');
+ });
+ });
+});
diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx
index 3a80686..ffdde25 100644
--- a/web-console/src/utils/general.tsx
+++ b/web-console/src/utils/general.tsx
@@ -19,6 +19,7 @@
import { Button, HTMLSelect, InputGroup, Intent } from '@blueprintjs/core';
import { IconNames } from '@blueprintjs/icons';
import FileSaver from 'file-saver';
+import hasOwnProp from 'has-own-prop';
import numeral from 'numeral';
import React from 'react';
import { Filter, FilterRender } from 'react-table';
@@ -165,7 +166,15 @@ export function groupBy<T, Q>(array: T[], keyFn: (x: T, index: number) => string
}
export function uniq(array: string[]): string[] {
- return Object.keys(lookupBy(array));
+ const seen: Record<string, boolean> = {};
+ return array.filter(s => {
+ if (hasOwnProp(seen, s)) {
+ return false;
+ } else {
+ seen[s] = true;
+ return true;
+ }
+ });
}
export function parseList(list: string): string[] {
@@ -253,11 +262,15 @@ export function filterMap<T, Q>(xs: T[], f: (x: T, i?: number) => Q | null | und
return (xs.map(f) as any).filter(Boolean);
}
-export function sortWithPrefixSuffix(things: string[], prefix: string[], suffix: string[]): string[] {
- const pre = things.filter((x) => prefix.includes(x)).sort();
- const mid = things.filter((x) => !prefix.includes(x) && !suffix.includes(x)).sort();
- const post = things.filter((x) => suffix.includes(x)).sort();
- return pre.concat(mid, post);
+export function alphanumericCompare(a: string, b: string): number {
+ return String(a).localeCompare(b, undefined, { numeric: true });
+}
+
+export function sortWithPrefixSuffix(things: string[], prefix: string[], suffix: string[], cmp: null | ((a: string, b: string) => number)): string[] {
+ const pre = uniq(prefix.filter((x) => things.includes(x)));
+ const mid = things.filter((x) => !prefix.includes(x) && !suffix.includes(x));
+ const post = uniq(suffix.filter((x) => things.includes(x)));
+ return pre.concat(cmp ? mid.sort(cmp) : mid, post);
}
// ----------------------------
diff --git a/web-console/src/utils/sampler.ts b/web-console/src/utils/sampler.ts
index 1996bf3..eeb28e3 100644
--- a/web-console/src/utils/sampler.ts
+++ b/web-console/src/utils/sampler.ts
@@ -19,7 +19,7 @@
import axios from 'axios';
import { getDruidErrorMessage } from './druid-query';
-import { filterMap, sortWithPrefixSuffix } from './general';
+import { alphanumericCompare, filterMap, sortWithPrefixSuffix } from './general';
import {
DimensionsSpec,
getEmptyTimestampSpec, getSpecType,
@@ -88,10 +88,13 @@ export function getSamplerType(spec: IngestionSpec): SamplerType {
return 'index';
}
-export function headerFromSampleResponse(sampleResponse: SampleResponse, ignoreColumn?: string): string[] {
- let columns = sortWithPrefixSuffix(dedupe(
- [].concat(...(filterMap(sampleResponse.data, s => s.parsed ? Object.keys(s.parsed) : null) as any))
- ).sort(), ['__time'], []);
+export function headerFromSampleResponse(sampleResponse: SampleResponse, ignoreColumn?: string, columnOrder?: string[]): string[] {
+ let columns = sortWithPrefixSuffix(
+ dedupe([].concat(...(filterMap(sampleResponse.data, s => s.parsed ? Object.keys(s.parsed) : null) as any))).sort(),
+ columnOrder || ['__time'],
+ [],
+ alphanumericCompare
+ );
if (ignoreColumn) {
columns = columns.filter(c => c !== ignoreColumn);
@@ -100,9 +103,9 @@ export function headerFromSampleResponse(sampleResponse: SampleResponse, ignoreC
return columns;
}
-export function headerAndRowsFromSampleResponse(sampleResponse: SampleResponse, ignoreColumn?: string, parsedOnly = false): HeaderAndRows {
+export function headerAndRowsFromSampleResponse(sampleResponse: SampleResponse, ignoreColumn?: string, columnOrder?: string[], parsedOnly = false): HeaderAndRows {
return {
- header: headerFromSampleResponse(sampleResponse, ignoreColumn),
+ header: headerFromSampleResponse(sampleResponse, ignoreColumn, columnOrder),
rows: parsedOnly ? sampleResponse.data.filter((d: any) => d.parsed) : sampleResponse.data
};
}
@@ -292,6 +295,7 @@ export async function sampleForTransform(spec: IngestionSpec, sampleStrategy: Sa
const ioConfig: IoConfig = makeSamplerIoConfig(deepGet(spec, 'ioConfig'), samplerType, sampleStrategy);
const parser: Parser = deepGet(spec, 'dataSchema.parser') || {};
const parseSpec: ParseSpec = deepGet(spec, 'dataSchema.parser.parseSpec') || {};
+ const parserColumns: string[] = deepGet(parseSpec, 'columns') || [];
const transforms: Transform[] = deepGet(spec, 'dataSchema.transformSpec.transforms') || [];
// Extra step to simulate auto detecting dimension with transforms
@@ -320,7 +324,7 @@ export async function sampleForTransform(spec: IngestionSpec, sampleStrategy: Sa
const sampleResponseHack = await postToSampler(sampleSpecHack, 'transform-pre');
- specialDimensionSpec.dimensions = dedupe(headerFromSampleResponse(sampleResponseHack, '__time').concat(transforms.map(t => t.name)));
+ specialDimensionSpec.dimensions = dedupe(headerFromSampleResponse(sampleResponseHack, '__time', ['__time'].concat(parserColumns)).concat(transforms.map(t => t.name)));
}
const sampleSpec: SampleSpec = {
@@ -354,6 +358,7 @@ export async function sampleForFilter(spec: IngestionSpec, sampleStrategy: Sampl
const ioConfig: IoConfig = makeSamplerIoConfig(deepGet(spec, 'ioConfig'), samplerType, sampleStrategy);
const parser: Parser = deepGet(spec, 'dataSchema.parser') || {};
const parseSpec: ParseSpec = deepGet(spec, 'dataSchema.parser.parseSpec') || {};
+ const parserColumns: string[] = deepGet(parser, 'columns') || [];
const transforms: Transform[] = deepGet(spec, 'dataSchema.transformSpec.transforms') || [];
const filter: any = deepGet(spec, 'dataSchema.transformSpec.filter');
@@ -383,7 +388,7 @@ export async function sampleForFilter(spec: IngestionSpec, sampleStrategy: Sampl
const sampleResponseHack = await postToSampler(sampleSpecHack, 'filter-pre');
- specialDimensionSpec.dimensions = dedupe(headerFromSampleResponse(sampleResponseHack, '__time').concat(transforms.map(t => t.name)));
+ specialDimensionSpec.dimensions = dedupe(headerFromSampleResponse(sampleResponseHack, '__time', ['__time'].concat(parserColumns)).concat(transforms.map(t => t.name)));
}
const sampleSpec: SampleSpec = {
diff --git a/web-console/src/views/load-data-view/load-data-view.scss b/web-console/src/views/load-data-view/load-data-view.scss
index 0e5a1b1..a565ded 100644
--- a/web-console/src/views/load-data-view/load-data-view.scss
+++ b/web-console/src/views/load-data-view/load-data-view.scss
@@ -61,7 +61,7 @@
&.tuning,
&.publish {
grid-gap: 20px 40px;
- grid-template-columns: 1fr 1fr 1fr;
+ grid-template-columns: 1fr 1fr 280px;
grid-template-areas:
"navi navi navi"
"main othr ctrl"
@@ -75,25 +75,25 @@
}
}
- .stage-nav {
+ .step-nav {
grid-area: navi;
white-space: nowrap;
overflow: auto;
padding: 0 5px;
- .stage-section {
+ .step-section {
display: inline-block;
vertical-align: top;
margin-right: 15px;
}
- .stage-nav-l1 {
+ .step-nav-l1 {
height: 25px;
font-weight: bold;
color: #eeeeee;
}
- .stage-nav-l2 {
+ .step-nav-l2 {
height: 30px;
}
}
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx
index 418e6a9..8883b03 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -137,17 +137,17 @@ function getTimestampSpec(headerAndRows: HeaderAndRows | null): TimestampSpec {
return timestampSpecs[0] || getEmptyTimestampSpec();
}
-type Stage = 'connect' | 'parser' | 'timestamp' | 'transform' | 'filter' | 'schema' | 'partition' | 'tuning' | 'publish' | 'json-spec' | 'loading';
-const STAGES: Stage[] = ['connect', 'parser', 'timestamp', 'transform', 'filter', 'schema', 'partition', 'tuning', 'publish', 'json-spec', 'loading'];
-
-const SECTIONS: { name: string, stages: Stage[] }[] = [
- { name: 'Connect and parse raw data', stages: ['connect', 'parser', 'timestamp'] },
- { name: 'Transform and configure schema', stages: ['transform', 'filter', 'schema'] },
- { name: 'Tune parameters', stages: ['partition', 'tuning', 'publish'] },
- { name: 'Verify and submit', stages: ['json-spec'] }
+type Step = 'connect' | 'parser' | 'timestamp' | 'transform' | 'filter' | 'schema' | 'partition' | 'tuning' | 'publish' | 'json-spec' | 'loading';
+const STEPS: Step[] = ['connect', 'parser', 'timestamp', 'transform', 'filter', 'schema', 'partition', 'tuning', 'publish', 'json-spec', 'loading'];
+
+const SECTIONS: { name: string, steps: Step[] }[] = [
+ { name: 'Connect and parse raw data', steps: ['connect', 'parser', 'timestamp'] },
+ { name: 'Transform and configure schema', steps: ['transform', 'filter', 'schema'] },
+ { name: 'Tune parameters', steps: ['partition', 'tuning', 'publish'] },
+ { name: 'Verify and submit', steps: ['json-spec'] }
];
-const VIEW_TITLE: Record<Stage, string> = {
+const VIEW_TITLE: Record<Step, string> = {
'connect': 'Connect',
'parser': 'Parse data',
'timestamp': 'Parse time',
@@ -168,7 +168,7 @@ export interface LoadDataViewProps extends React.Props<any> {
}
export interface LoadDataViewState {
- stage: Stage;
+ step: Step;
spec: IngestionSpec;
cacheKey: string | undefined;
// dialogs / modals
@@ -230,7 +230,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
let spec = parseJson(String(localStorageGet(LocalStorageKeys.INGESTION_SPEC)));
if (!spec || typeof spec !== 'object') spec = {};
this.state = {
- stage: 'connect',
+ step: 'connect',
spec,
cacheKey: undefined,
@@ -283,15 +283,15 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
componentDidMount(): void {
this.getOverlordModules();
if (this.props.initTaskId) {
- this.updateStage('loading');
+ this.updateStep('loading');
this.getTaskJson();
} else if (this.props.initSupervisorId) {
- this.updateStage('loading');
+ this.updateStep('loading');
this.getSupervisorJson();
} else {
- this.updateStage('connect');
+ this.updateStep('connect');
}
}
@@ -312,13 +312,13 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
this.setState({ overlordModules });
}
- private updateStage = (newStage: Stage) => {
- this.doQueryForStage(newStage);
- this.setState({ stage: newStage });
+ private updateStep = (newStep: Step) => {
+ this.doQueryForStep(newStep);
+ this.setState({ step: newStep });
}
- doQueryForStage(stage: Stage): any {
- switch (stage) {
+ doQueryForStep(step: Step): any {
+ switch (step) {
case 'connect': return this.queryForConnect(true);
case 'parser': return this.queryForParser(true);
case 'timestamp': return this.queryForTimestamp(true);
@@ -338,51 +338,51 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
}
render() {
- const { stage, spec } = this.state;
+ const { step, spec } = this.state;
if (!Object.keys(spec).length && !this.props.initSupervisorId && !this.props.initTaskId) {
return <div className={classNames('load-data-view', 'app-view', 'init')}>
- {this.renderInitStage()}
+ {this.renderInitStep()}
</div>;
}
- return <div className={classNames('load-data-view', 'app-view', stage)}>
+ return <div className={classNames('load-data-view', 'app-view', step)}>
{this.renderStepNav()}
- {stage === 'connect' && this.renderConnectStage()}
- {stage === 'parser' && this.renderParserStage()}
- {stage === 'timestamp' && this.renderTimestampStage()}
+ {step === 'connect' && this.renderConnectStep()}
+ {step === 'parser' && this.renderParserStep()}
+ {step === 'timestamp' && this.renderTimestampStep()}
- {stage === 'transform' && this.renderTransformStage()}
- {stage === 'filter' && this.renderFilterStage()}
- {stage === 'schema' && this.renderSchemaStage()}
+ {step === 'transform' && this.renderTransformStep()}
+ {step === 'filter' && this.renderFilterStep()}
+ {step === 'schema' && this.renderSchemaStep()}
- {stage === 'partition' && this.renderPartitionStage()}
- {stage === 'tuning' && this.renderTuningStage()}
- {stage === 'publish' && this.renderPublishStage()}
+ {step === 'partition' && this.renderPartitionStep()}
+ {step === 'tuning' && this.renderTuningStep()}
+ {step === 'publish' && this.renderPublishStep()}
- {stage === 'json-spec' && this.renderJsonSpecStage()}
- {stage === 'loading' && this.renderLoading()}
+ {step === 'json-spec' && this.renderJsonSpecStep()}
+ {step === 'loading' && this.renderLoading()}
{this.renderResetConfirm()}
</div>;
}
renderStepNav() {
- const { stage } = this.state;
+ const { step } = this.state;
- return <div className={classNames(Classes.TABS, 'stage-nav')}>
+ return <div className={classNames(Classes.TABS, 'step-nav')}>
{SECTIONS.map(section => (
- <div className="stage-section" key={section.name}>
- <div className="stage-nav-l1">
+ <div className="step-section" key={section.name}>
+ <div className="step-nav-l1">
{section.name}
</div>
- <ButtonGroup className="stage-nav-l2">
- {section.stages.map((s) => (
+ <ButtonGroup className="step-nav-l2">
+ {section.steps.map((s) => (
<Button
className={s}
key={s}
- active={s === stage}
- onClick={() => this.updateStage(s)}
+ active={s === step}
+ onClick={() => this.updateStep(s)}
icon={s === 'json-spec' && IconNames.MANUALLY_ENTERED_DATA}
text={VIEW_TITLE[s]}
/>
@@ -393,32 +393,32 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
</div>;
}
- renderNextBar(options: { nextStage?: Stage, disabled?: boolean; onNextStage?: () => void, onPrevStage?: () => void, prevLabel?: string }) {
- const { disabled, onNextStage, onPrevStage, prevLabel } = options;
- const { stage } = this.state;
- const nextStage = options.nextStage || STAGES[STAGES.indexOf(stage) + 1] || STAGES[0];
+ renderNextBar(options: { nextStep?: Step, disabled?: boolean; onNextStep?: () => void, onPrevStep?: () => void, prevLabel?: string }) {
+ const { disabled, onNextStep, onPrevStep, prevLabel } = options;
+ const { step } = this.state;
+ const nextStep = options.nextStep || STEPS[STEPS.indexOf(step) + 1] || STEPS[0];
return <div className="next-bar">
{
- onPrevStage &&
+ onPrevStep &&
<Button
className="prev"
icon={IconNames.UNDO}
text={prevLabel}
- onClick={onPrevStage}
+ onClick={onPrevStep}
/>
}
<Button
- text={`Next: ${VIEW_TITLE[nextStage]}`}
+ text={`Next: ${VIEW_TITLE[nextStep]}`}
rightIcon={IconNames.ARROW_RIGHT}
intent={Intent.PRIMARY}
disabled={disabled}
onClick={() => {
if (disabled) return;
- if (onNextStage) onNextStage();
+ if (onNextStep) onNextStep();
setTimeout(() => {
- this.updateStage(nextStage);
+ this.updateStep(nextStep);
}, 10);
}}
/>
@@ -432,7 +432,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
spec: getBlankSpec(comboType)
});
setTimeout(() => {
- this.updateStage('connect');
+ this.updateStep('connect');
}, 10);
}
@@ -458,7 +458,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
</Card>;
}
- renderInitStage() {
+ renderInitStep() {
const { goToTask } = this.props;
const { overlordModuleNeededMessage } = this.state;
@@ -554,7 +554,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
});
}
- renderConnectStage() {
+ renderConnectStep() {
const { spec, inputQueryState, sampleStrategy } = this.state;
const specType = getSpecType(spec);
const ioConfig: IoConfig = deepGet(spec, 'ioConfig') || EMPTY_OBJECT;
@@ -563,7 +563,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
let mainFill: JSX.Element | string = '';
if (inputQueryState.isInit()) {
mainFill = <CenterMessage>
- Please fill out the fields on the right sidebar to get started.
+ Please fill out the fields on the right sidebar to get started <Icon icon={IconNames.ARROW_RIGHT}/>
</CenterMessage>;
} else if (inputQueryState.isLoading()) {
@@ -642,12 +642,12 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
</div>
{this.renderNextBar({
disabled: !inputQueryState.data,
- onNextStage: () => {
+ onNextStep: () => {
if (!inputQueryState.data) return;
this.updateSpec(fillDataSourceName(fillParser(spec, inputQueryState.data)));
},
prevLabel: 'Restart',
- onPrevStage: () => this.setState({ showResetConfirm: true })
+ onPrevStep: () => this.setState({ showResetConfirm: true })
})}
</>;
}
@@ -658,6 +658,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
const { spec, sampleStrategy, cacheKey } = this.state;
const ioConfig: IoConfig = deepGet(spec, 'ioConfig') || EMPTY_OBJECT;
const parser: Parser = deepGet(spec, 'dataSchema.parser') || EMPTY_OBJECT;
+ const parserColumns: string[] = deepGet(parser, 'parseSpec.columns') || [];
let issue: string | null = null;
if (issueWithIoConfig(ioConfig)) {
@@ -690,12 +691,12 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
this.setState({
cacheKey: sampleResponse.cacheKey,
parserQueryState: new QueryState({
- data: headerAndRowsFromSampleResponse(sampleResponse, '__time')
+ data: headerAndRowsFromSampleResponse(sampleResponse, '__time', parserColumns)
})
});
}
- renderParserStage() {
+ renderParserStep() {
const { spec, columnFilter, specialColumnsOnly, parserQueryState, selectedFlattenField } = this.state;
const parseSpec: ParseSpec = deepGet(spec, 'dataSchema.parser.parseSpec') || EMPTY_OBJECT;
const flattenFields: FlattenField[] = deepGet(spec, 'dataSchema.parser.parseSpec.flattenSpec.fields') || EMPTY_ARRAY;
@@ -706,7 +707,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
let mainFill: JSX.Element | string = '';
if (parserQueryState.isInit()) {
mainFill = <CenterMessage>
- Please enter the parser details on the right
+ Please enter the parser details on the right <Icon icon={IconNames.ARROW_RIGHT}/>
</CenterMessage>;
} else if (parserQueryState.isLoading()) {
@@ -802,7 +803,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
</div>
{this.renderNextBar({
disabled: !parserQueryState.data,
- onNextStage: () => {
+ onNextStep: () => {
if (!parserQueryState.data) return;
const possibleTimestampSpec = getTimestampSpec(parserQueryState.data);
if (possibleTimestampSpec) {
@@ -898,6 +899,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
const { spec, sampleStrategy, cacheKey } = this.state;
const ioConfig: IoConfig = deepGet(spec, 'ioConfig') || EMPTY_OBJECT;
const parser: Parser = deepGet(spec, 'dataSchema.parser') || EMPTY_OBJECT;
+ const parserColumns: string[] = deepGet(parser, 'parseSpec.columns') || [];
const timestampSpec = deepGet(spec, 'dataSchema.parser.parseSpec.timestampSpec') || EMPTY_OBJECT;
let issue: string | null = null;
@@ -932,14 +934,14 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
cacheKey: sampleResponse.cacheKey,
timestampQueryState: new QueryState({
data: {
- headerAndRows: headerAndRowsFromSampleResponse(sampleResponse),
+ headerAndRows: headerAndRowsFromSampleResponse(sampleResponse, undefined, ['__time'].concat(parserColumns)),
timestampSpec
}
})
});
}
- renderTimestampStage() {
+ renderTimestampStep() {
const { spec, columnFilter, specialColumnsOnly, timestampQueryState } = this.state;
const parseSpec: ParseSpec = deepGet(spec, 'dataSchema.parser.parseSpec') || EMPTY_OBJECT;
const timestampSpec: TimestampSpec = deepGet(spec, 'dataSchema.parser.parseSpec.timestampSpec') || EMPTY_OBJECT;
@@ -950,7 +952,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
let mainFill: JSX.Element | string = '';
if (timestampQueryState.isInit()) {
mainFill = <CenterMessage>
- Please enter the timestamp column details on the right
+ Please enter the timestamp column details on the right <Icon icon={IconNames.ARROW_RIGHT}/>
</CenterMessage>;
} else if (timestampQueryState.isLoading()) {
@@ -1056,6 +1058,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
const { spec, sampleStrategy, cacheKey } = this.state;
const ioConfig: IoConfig = deepGet(spec, 'ioConfig') || EMPTY_OBJECT;
const parser: Parser = deepGet(spec, 'dataSchema.parser') || EMPTY_OBJECT;
+ const parserColumns: string[] = deepGet(parser, 'parseSpec.columns') || [];
let issue: string | null = null;
if (issueWithIoConfig(ioConfig)) {
@@ -1088,12 +1091,12 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
this.setState({
cacheKey: sampleResponse.cacheKey,
transformQueryState: new QueryState({
- data: headerAndRowsFromSampleResponse(sampleResponse)
+ data: headerAndRowsFromSampleResponse(sampleResponse, undefined, ['__time'].concat(parserColumns))
})
});
}
- renderTransformStage() {
+ renderTransformStep() {
const { spec, columnFilter, specialColumnsOnly, transformQueryState, selectedTransformIndex } = this.state;
const transforms: Transform[] = deepGet(spec, 'dataSchema.transformSpec.transforms') || EMPTY_ARRAY;
@@ -1175,7 +1178,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
</div>
{this.renderNextBar({
disabled: !transformQueryState.data,
- onNextStage: () => {
+ onNextStep: () => {
if (!transformQueryState.data) return;
this.updateSpec(updateSchemaWithSample(spec, transformQueryState.data, 'specific', true));
}
@@ -1260,6 +1263,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
const { spec, sampleStrategy, cacheKey } = this.state;
const ioConfig: IoConfig = deepGet(spec, 'ioConfig') || EMPTY_OBJECT;
const parser: Parser = deepGet(spec, 'dataSchema.parser') || EMPTY_OBJECT;
+ const parserColumns: string[] = deepGet(parser, 'parseSpec.columns') || [];
let issue: string | null = null;
if (issueWithIoConfig(ioConfig)) {
@@ -1292,7 +1296,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
this.setState({
cacheKey: sampleResponse.cacheKey,
filterQueryState: new QueryState({
- data: headerAndRowsFromSampleResponse(sampleResponse, undefined, true)
+ data: headerAndRowsFromSampleResponse(sampleResponse, undefined, ['__time'].concat(parserColumns), true)
})
});
}
@@ -1302,7 +1306,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
return dimensionFilters;
});
- renderFilterStage() {
+ renderFilterStep() {
const { spec, columnFilter, filterQueryState, selectedFilter, selectedFilterIndex, showGlobalFilter } = this.state;
const parseSpec: ParseSpec = deepGet(spec, 'dataSchema.parser.parseSpec') || EMPTY_OBJECT;
const dimensionFilters = this.getMemoizedDimensionFiltersFromSpec(spec);
@@ -1515,6 +1519,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
const { spec, sampleStrategy, cacheKey } = this.state;
const ioConfig: IoConfig = deepGet(spec, 'ioConfig') || EMPTY_OBJECT;
const parser: Parser = deepGet(spec, 'dataSchema.parser') || EMPTY_OBJECT;
+ const parserColumns: string[] = deepGet(parser, 'parseSpec.columns') || [];
const metricsSpec: MetricSpec[] = deepGet(spec, 'dataSchema.metricsSpec') || EMPTY_ARRAY;
const dimensionsSpec: DimensionsSpec = deepGet(spec, 'dataSchema.parser.parseSpec.dimensionsSpec') || EMPTY_OBJECT;
@@ -1550,7 +1555,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
cacheKey: sampleResponse.cacheKey,
schemaQueryState: new QueryState({
data: {
- headerAndRows: headerAndRowsFromSampleResponse(sampleResponse),
+ headerAndRows: headerAndRowsFromSampleResponse(sampleResponse, undefined, ['__time'].concat(parserColumns)),
dimensionsSpec,
metricsSpec
}
@@ -1558,7 +1563,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
});
}
- renderSchemaStage() {
+ renderSchemaStep() {
const { spec, columnFilter, schemaQueryState, selectedDimensionSpec, selectedDimensionSpecIndex, selectedMetricSpec, selectedMetricSpecIndex } = this.state;
const rollup: boolean = Boolean(deepGet(spec, 'dataSchema.granularitySpec.rollup'));
const somethingSelected = Boolean(selectedDimensionSpec || selectedMetricSpec);
@@ -1917,7 +1922,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
// ==================================================================
- renderPartitionStage() {
+ renderPartitionStep() {
const { spec } = this.state;
const tuningConfig: TuningConfig = deepGet(spec, 'tuningConfig') || EMPTY_OBJECT;
const granularitySpec: GranularitySpec = deepGet(spec, 'dataSchema.granularitySpec') || EMPTY_OBJECT;
@@ -1976,7 +1981,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
// ==================================================================
- renderTuningStage() {
+ renderTuningStep() {
const { spec } = this.state;
const ioConfig: IoConfig = deepGet(spec, 'ioConfig') || EMPTY_OBJECT;
const tuningConfig: TuningConfig = deepGet(spec, 'tuningConfig') || EMPTY_OBJECT;
@@ -2061,7 +2066,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
// ==================================================================
- renderPublishStage() {
+ renderPublishStep() {
const { spec } = this.state;
return <>
@@ -2150,7 +2155,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
try {
const resp = await axios.get(`/druid/indexer/v1/supervisor/${initSupervisorId}`);
this.updateSpec(normalizeSpecType(resp.data));
- this.updateStage('json-spec');
+ this.updateStep('json-spec');
} catch (e) {
AppToaster.show({
message: `Failed to get supervisor spec: ${getDruidErrorMessage(e)}`,
@@ -2165,7 +2170,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
try {
const resp = await axios.get(`/druid/indexer/v1/task/${initTaskId}`);
this.updateSpec(normalizeSpecType(resp.data.payload.spec));
- this.updateStage('json-spec');
+ this.updateStep('json-spec');
} catch (e) {
AppToaster.show({
message: `Failed to get task spec: ${getDruidErrorMessage(e)}`,
@@ -2178,7 +2183,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
return <Loader loading/>;
}
- renderJsonSpecStage() {
+ renderJsonSpecStep() {
const { goToTask } = this.props;
const { spec } = this.state;
diff --git a/web-console/src/views/load-data-view/schema-table/schema-table.tsx b/web-console/src/views/load-data-view/schema-table/schema-table.tsx
index 1b6f96f..75243d3 100644
--- a/web-console/src/views/load-data-view/schema-table/schema-table.tsx
+++ b/web-console/src/views/load-data-view/schema-table/schema-table.tsx
@@ -21,7 +21,7 @@ import React from 'react';
import ReactTable from 'react-table';
import { TableCell } from '../../../components';
-import { caseInsensitiveContains, filterMap, sortWithPrefixSuffix } from '../../../utils';
+import { alphanumericCompare, caseInsensitiveContains, filterMap, sortWithPrefixSuffix } from '../../../utils';
import {
DimensionSpec,
DimensionsSpec,
@@ -57,7 +57,7 @@ export class SchemaTable extends React.PureComponent<SchemaTableProps> {
const { sampleBundle, columnFilter, selectedDimensionSpecIndex, selectedMetricSpecIndex, onDimensionOrMetricSelect } = this.props;
const { headerAndRows, dimensionsSpec, metricsSpec } = sampleBundle;
- const dimensionMetricSortedHeader = sortWithPrefixSuffix(headerAndRows.header, ['__time'], metricsSpec.map(getMetricSpecName));
+ const dimensionMetricSortedHeader = sortWithPrefixSuffix(headerAndRows.header, ['__time'], metricsSpec.map(getMetricSpecName), null);
return <ReactTable
className="schema-table -striped -highlight"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org