You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by cw...@apache.org on 2019/05/19 04:04:50 UTC
[incubator-druid] branch 0.15.0-incubating updated: Data loader
fixes (#7672)
This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch 0.15.0-incubating
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/0.15.0-incubating by this push:
new ba6caf5 Data loader fixes (#7672)
ba6caf5 is described below
commit ba6caf5bfd9bd1b2c763457638bc4bca6dbdfb85
Author: Vadim Ogievetsky <va...@gmail.com>
AuthorDate: Sat May 18 21:04:42 2019 -0700
Data loader fixes (#7672)
* add ability to submit 'other' task
* copy changes, fix sort
* lets call it dialog
* update name to Google Cloud Storage
---
web-console/src/console-application.tsx | 7 ++++--
web-console/src/views/load-data-view.scss | 1 +
web-console/src/views/load-data-view.tsx | 41 +++++++++++++++++++++++++------
web-console/src/views/tasks-view.tsx | 35 +++++++++++++-------------
4 files changed, 58 insertions(+), 26 deletions(-)
diff --git a/web-console/src/console-application.tsx b/web-console/src/console-application.tsx
index 4f10977..58763b2 100644
--- a/web-console/src/console-application.tsx
+++ b/web-console/src/console-application.tsx
@@ -102,6 +102,7 @@ export class ConsoleApplication extends React.Component<ConsoleApplicationProps,
private loadDataViewSeed: LoadDataViewSeed | null;
private taskId: string | null;
+ private openDialog: string | null;
private datasource: string | null;
private onlyUnavailable: boolean | null;
private initSql: string | null;
@@ -152,6 +153,7 @@ export class ConsoleApplication extends React.Component<ConsoleApplicationProps,
setTimeout(() => {
this.loadDataViewSeed = null;
this.taskId = null;
+ this.openDialog = null;
this.datasource = null;
this.onlyUnavailable = null;
this.initSql = null;
@@ -165,8 +167,9 @@ export class ConsoleApplication extends React.Component<ConsoleApplicationProps,
this.resetInitialsWithDelay();
}
- private goToTask = (taskId: string | null) => {
+ private goToTask = (taskId: string | null, openDialog: string | null = null) => {
this.taskId = taskId;
+ this.openDialog = openDialog;
window.location.hash = 'tasks';
this.resetInitialsWithDelay();
}
@@ -224,7 +227,7 @@ export class ConsoleApplication extends React.Component<ConsoleApplicationProps,
private wrappedTasksView = () => {
const { noSqlMode } = this.state;
- return this.wrapInViewContainer('tasks', <TasksView taskId={this.taskId} goToSql={this.goToSql} goToMiddleManager={this.goToMiddleManager} goToLoadDataView={this.goToLoadDataView} noSqlMode={noSqlMode}/>, true);
+ return this.wrapInViewContainer('tasks', <TasksView taskId={this.taskId} openDialog={this.openDialog} goToSql={this.goToSql} goToMiddleManager={this.goToMiddleManager} goToLoadDataView={this.goToLoadDataView} noSqlMode={noSqlMode}/>, true);
}
private wrappedServersView = () => {
diff --git a/web-console/src/views/load-data-view.scss b/web-console/src/views/load-data-view.scss
index 050672d..5e08831 100644
--- a/web-console/src/views/load-data-view.scss
+++ b/web-console/src/views/load-data-view.scss
@@ -46,6 +46,7 @@
.cards {
.bp3-card {
display: inline-block;
+ vertical-align: top;
width: 250px;
height: 140px;
margin-right: 15px;
diff --git a/web-console/src/views/load-data-view.tsx b/web-console/src/views/load-data-view.tsx
index 22a0dbb..bbd9e76 100644
--- a/web-console/src/views/load-data-view.tsx
+++ b/web-console/src/views/load-data-view.tsx
@@ -137,7 +137,7 @@ const VIEW_TITLE: Record<Stage, string> = {
export interface LoadDataViewProps extends React.Props<any> {
seed: LoadDataViewSeed | null;
- goToTask: (taskId: string | null) => void;
+ goToTask: (taskId: string | null, openDialog?: string | null) => void;
}
export interface LoadDataViewState {
@@ -370,6 +370,7 @@ export class LoadDataView extends React.Component<LoadDataViewProps, LoadDataVie
}
renderInitStage() {
+ const { goToTask } = this.props;
const showStreaming = false;
return <>
@@ -378,8 +379,10 @@ export class LoadDataView extends React.Component<LoadDataViewProps, LoadDataVie
</div>
<Callout intent={Intent.SUCCESS} icon={IconNames.INFO_SIGN}>
- Welcome to the Druid data loader.
- This project is under active development and we plan to support many other sources of raw data, including stream hubs such as Apache Kafka and AWS Kinesis, in the next few releases.
+ Welcome to the Apache Druid graphical data loader.
+ This feature is under active development and currently only supports Druid's native batch ingestion.
+ We plan to continue building this out, including support for Druid's Apache Kafka, Apache Hadoop, and AWS Kinesis based ingestion methods, over the next few releases.
+ Until then, you can load from these and any other Druid ingestion source by clicking on <Code>Other</Code>.
</Callout>
{
@@ -398,10 +401,18 @@ export class LoadDataView extends React.Component<LoadDataViewProps, LoadDataVie
<div className="cards">
<Card interactive onClick={() => this.initWith({ type: 'index_parallel', firehoseType: 'http' })}>HTTP(s)</Card>
<Card interactive onClick={() => this.initWith({ type: 'index_parallel', firehoseType: 'static-s3' })}>AWS S3</Card>
- <Card interactive onClick={() => this.initWith({ type: 'index_parallel', firehoseType: 'static-google-blobstore' })}>Google Blobstore</Card>
+ <Card interactive onClick={() => this.initWith({ type: 'index_parallel', firehoseType: 'static-google-blobstore' })}>Google Cloud Storage</Card>
<Card interactive onClick={() => this.initWith({ type: 'index_parallel', firehoseType: 'local' })}>Local disk</Card>
</div>
</div>
+
+ <div className="section">
+ <div className="section-title">Raw spec</div>
+ <div className="cards">
+ <Card interactive onClick={() => goToTask(null, 'supervisor')}>Other (streaming)</Card>
+ <Card interactive onClick={() => goToTask(null, 'task')}>Other (batch)</Card>
+ </div>
+ </div>
</>;
}
@@ -1179,6 +1190,22 @@ export class LoadDataView extends React.Component<LoadDataViewProps, LoadDataVie
Click "Preview" to see the result of any specified transforms.
</p>
</Callout>
+ {
+ Boolean(transformQueryState.error && transforms.length) &&
+ <FormGroup>
+ <Button
+ icon={IconNames.EDIT}
+ text="Edit last added transform"
+ intent={Intent.PRIMARY}
+ onClick={() => {
+ this.setState({
+ selectedTransformIndex: transforms.length - 1,
+ selectedTransform: transforms[transforms.length - 1]
+ });
+ }}
+ />
+ </FormGroup>
+ }
{this.renderTransformControls()}
<Button
text="Preview"
@@ -1729,7 +1756,7 @@ export class LoadDataView extends React.Component<LoadDataViewProps, LoadDataVie
<Switch
checked={dimensionMode === 'specific'}
onChange={() => this.setState({ newDimensionMode: dimensionMode === 'specific' ? 'auto-detect' : 'specific' })}
- label="Set dimensions and metrics"
+ label="Explicitly specify dimension list"
/>
<Popover
content={
@@ -1864,8 +1891,8 @@ export class LoadDataView extends React.Component<LoadDataViewProps, LoadDataVie
<p>
{
autoDetect ?
- 'Are you sure you don’t want to set the dimensions and metrics explicitly?' :
- 'Are you sure you want to set dimensions and metrics explicitly?'
+ `Are you sure you don't want to explicitly specify a dimension list?` :
+ `Are you sure you want to explicitly specify a dimension list?`
}
</p>
<p>
diff --git a/web-console/src/views/tasks-view.tsx b/web-console/src/views/tasks-view.tsx
index 8acee10..6aee820 100644
--- a/web-console/src/views/tasks-view.tsx
+++ b/web-console/src/views/tasks-view.tsx
@@ -49,6 +49,7 @@ const taskTableColumns: string[] = ['Task ID', 'Type', 'Datasource', 'Created ti
export interface TasksViewProps extends React.Props<any> {
taskId: string | null;
+ openDialog: string | null;
goToSql: (initSql: string) => void;
goToMiddleManager: (middleManager: string) => void;
goToLoadDataView: () => void;
@@ -117,7 +118,7 @@ export class TasksView extends React.Component<TasksViewProps, TasksViewState> {
private taskQueryManager: QueryManager<string, TaskQueryResultRow[]>;
private supervisorTableColumnSelectionHandler: TableColumnSelectionHandler;
private taskTableColumnSelectionHandler: TableColumnSelectionHandler;
- static statusRanking = {RUNNING: 4, PENDING: 3, WAITING: 2, SUCCESS: 1, FAILED: 1};
+ static statusRanking: Record<string, number> = {RUNNING: 4, PENDING: 3, WAITING: 2, SUCCESS: 1, FAILED: 1};
constructor(props: TasksViewProps, context: any) {
super(props, context);
@@ -139,8 +140,8 @@ export class TasksView extends React.Component<TasksViewProps, TasksViewState> {
killTaskId: null,
- supervisorSpecDialogOpen: false,
- taskSpecDialogOpen: false,
+ supervisorSpecDialogOpen: props.openDialog === 'supervisor',
+ taskSpecDialogOpen: props.openDialog === 'task',
initSpec: null,
alertErrorMsg: null,
@@ -600,7 +601,7 @@ ORDER BY "rank" DESC, "created_time" DESC`);
Header: 'Status',
id: 'status',
width: 110,
- accessor: (row) => { return {status: row.status, created_time: row.created_time}; },
+ accessor: row => ({ status: row.status, created_time: row.created_time, toString: () => row.status }),
Cell: row => {
if (row.aggregated) return '';
const { status, location } = row.original;
@@ -617,20 +618,20 @@ ORDER BY "rank" DESC, "created_time" DESC`);
{errorMsg && <a onClick={() => this.setState({ alertErrorMsg: errorMsg })} title={errorMsg}> ?</a>}
</span>;
},
- PivotValue: (opt) => {
- const { subRows, value } = opt;
- if (!subRows || !subRows.length) return '';
- return `${subRows[0]._original['status']} (${subRows.length})`;
- },
- Aggregated: (opt: any) => {
- const { subRows, column } = opt;
- const previewValues = subRows.filter((d: any) => typeof d[column.id] !== 'undefined').map((row: any) => row._original[column.id]);
- const previewCount = countBy(previewValues);
- return <span>{Object.keys(previewCount).sort().map(v => `${v} (${previewCount[v]})`).join(', ')}</span>;
- },
sortMethod: (d1, d2) => {
- const statusRanking: any = TasksView.statusRanking;
- return statusRanking[d1.status] - statusRanking[d2.status] || d1.created_time.localeCompare(d2.created_time);
+ const typeofD1 = typeof d1;
+ const typeofD2 = typeof d2;
+ if (typeofD1 !== typeofD2) return 0;
+ switch (typeofD1) {
+ case 'string':
+ return TasksView.statusRanking[d1] - TasksView.statusRanking[d2];
+
+ case 'object':
+ return TasksView.statusRanking[d1.status] - TasksView.statusRanking[d2.status] || d1.created_time.localeCompare(d2.created_time);
+
+ default:
+ return 0;
+ }
},
filterMethod: (filter: Filter, row: any) => {
return booleanCustomTableFilter(filter, row.status.status);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org