You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by vo...@apache.org on 2020/10/01 22:14:56 UTC
[druid] branch master updated: Web console reindexing E2E test
(#10453)
This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 7385af0 Web console reindexing E2E test (#10453)
7385af0 is described below
commit 7385af027250440ff66573789fc0543ee056777f
Author: Chi Cao Minh <ch...@imply.io>
AuthorDate: Thu Oct 1 15:14:41 2020 -0700
Web console reindexing E2E test (#10453)
Add an E2E test for the web console workflow of reindexing a Druid
datasource to change the secondary partitioning type. The new test
changes dynamic to single dim partitions since the autocompaction test
already does dynamic to hashed partitions.
Also, run the web console E2E tests in parallel to reduce CI time and
change naming convention for test datasources to make it easier to map
them to the corresponding test run.
Main changes:
1) web-consolee2e-tests/reindexing.spec.ts
- new E2E test
2) web-console/e2e-tests/component/load-data/data-connector/reindex.ts
- new data loader connector for druid input source
3) web-console/e2e-tests/component/load-data/config/partition.ts
- move partition spec definitions from compaction.ts
- add new single dim partition spec definition
---
web-console/e2e-tests/auto-compaction.spec.ts | 31 ++---
.../e2e-tests/component/datasources/compaction.ts | 41 +-----
.../e2e-tests/component/datasources/overview.ts | 20 ++-
.../component/load-data/config/partition.ts | 70 ++++++++++
.../load-data/data-connector/data-connector.ts | 9 ++
.../load-data/data-connector/local-file.ts | 33 +++--
.../data-connector/{local-file.ts => reindex.ts} | 39 +++---
.../e2e-tests/component/load-data/data-loader.ts | 66 ++++-----
web-console/e2e-tests/component/query/overview.ts | 15 +-
.../{tutorial-batch.spec.ts => reindexing.spec.ts} | 154 ++++++++++-----------
web-console/e2e-tests/tutorial-batch.spec.ts | 22 +--
web-console/e2e-tests/util/druid.ts | 23 ++-
web-console/e2e-tests/util/playwright.ts | 59 ++++++++
web-console/package.json | 2 +-
14 files changed, 345 insertions(+), 239 deletions(-)
diff --git a/web-console/e2e-tests/auto-compaction.spec.ts b/web-console/e2e-tests/auto-compaction.spec.ts
index edcf6a8..512c272 100644
--- a/web-console/e2e-tests/auto-compaction.spec.ts
+++ b/web-console/e2e-tests/auto-compaction.spec.ts
@@ -17,19 +17,18 @@
*/
import axios from 'axios';
-import { execSync } from 'child_process';
import path from 'path';
import * as playwright from 'playwright-core';
-import { v4 as uuid } from 'uuid';
import { CompactionConfig } from './component/datasources/compaction';
-import { CompactionHashPartitionsSpec } from './component/datasources/compaction';
import { Datasource } from './component/datasources/datasource';
import { DatasourcesOverview } from './component/datasources/overview';
+import { HashedPartitionsSpec } from './component/load-data/config/partition';
import { saveScreenshotIfError } from './util/debug';
import { COORDINATOR_URL } from './util/druid';
-import { DRUID_DIR } from './util/druid';
+import { DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR } from './util/druid';
import { UNIFIED_CONSOLE_URL } from './util/druid';
+import { runIndexTask } from './util/druid';
import { createBrowserNormal as createBrowser } from './util/playwright';
import { createPage } from './util/playwright';
import { retryIfJestAssertionError } from './util/retry';
@@ -57,17 +56,18 @@ describe('Auto-compaction', () => {
});
it('Compacts segments from dynamic to hash partitions', async () => {
- const datasourceName = uuid();
+ const testName = 'autocompaction-dynamic-to-hash-';
+ const datasourceName = testName + new Date().toISOString();
loadInitialData(datasourceName);
- await saveScreenshotIfError('auto-compaction-', page, async () => {
+ await saveScreenshotIfError(testName, page, async () => {
const uncompactedNumSegment = 3;
const numRow = 1412;
await validateDatasourceStatus(page, datasourceName, uncompactedNumSegment, numRow);
const compactionConfig = new CompactionConfig({
skipOffsetFromLatest: 'PT0S',
- partitionsSpec: new CompactionHashPartitionsSpec({
+ partitionsSpec: new HashedPartitionsSpec({
numShards: null,
}),
});
@@ -88,25 +88,14 @@ describe('Auto-compaction', () => {
});
function loadInitialData(datasourceName: string) {
- const postIndexTask = path.join(DRUID_DIR, 'examples', 'bin', 'post-index-task');
const ingestionSpec = path.join(
- DRUID_DIR,
- 'examples',
- 'quickstart',
- 'tutorial',
+ DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR,
'compaction-init-index.json',
);
const setDatasourceName = `s/compaction-tutorial/${datasourceName}/`;
const setIntervals = 's|2015-09-12/2015-09-13|2015-09-12/2015-09-12T02:00|'; // shorten to reduce test duration
- execSync(
- `${postIndexTask} \
- --file <(sed -e '${setDatasourceName}' -e '${setIntervals}' ${ingestionSpec}) \
- --url ${COORDINATOR_URL}`,
- {
- shell: 'bash',
- timeout: 3 * 60 * 1000,
- },
- );
+ const sedCommands = [setDatasourceName, setIntervals];
+ runIndexTask(ingestionSpec, sedCommands);
}
async function validateDatasourceStatus(
diff --git a/web-console/e2e-tests/component/datasources/compaction.ts b/web-console/e2e-tests/component/datasources/compaction.ts
index c22285d..9463620 100644
--- a/web-console/e2e-tests/component/datasources/compaction.ts
+++ b/web-console/e2e-tests/component/datasources/compaction.ts
@@ -16,44 +16,7 @@
* limitations under the License.
*/
-import * as playwright from 'playwright-core';
-
-/* tslint:disable max-classes-per-file */
-
-const PARTITIONING_TYPE = 'Partitioning type';
-
-interface CompactionPartitionsSpec {
- readonly type: string;
- apply(page: playwright.Page): Promise<void>;
-}
-
-export class CompactionHashPartitionsSpec implements CompactionPartitionsSpec {
- readonly type: string;
-
- constructor(props: CompactionHashPartitionsSpecProps) {
- Object.assign(this, props);
- this.type = 'hashed';
- }
-
- async apply(page: playwright.Page): Promise<void> {
- await setInput(page, PARTITIONING_TYPE, this.type);
- if (this.numShards != null) {
- await setInput(page, 'Num shards', String(this.numShards));
- }
- }
-}
-
-async function setInput(page: playwright.Page, label: string, value: string): Promise<void> {
- const input = await page.$(`//*[text()="${label}"]/following-sibling::div//input`);
- await input!.fill('');
- await input!.type(value);
-}
-
-interface CompactionHashPartitionsSpecProps {
- readonly numShards: number | null;
-}
-
-export interface CompactionHashPartitionsSpec extends CompactionHashPartitionsSpecProps {}
+import { PartitionsSpec } from '../load-data/config/partition';
/**
* Datasource compaction configuration
@@ -66,7 +29,7 @@ export class CompactionConfig {
interface CompactionConfigProps {
readonly skipOffsetFromLatest: string;
- readonly partitionsSpec: CompactionPartitionsSpec;
+ readonly partitionsSpec: PartitionsSpec;
}
export interface CompactionConfig extends CompactionConfigProps {}
diff --git a/web-console/e2e-tests/component/datasources/overview.ts b/web-console/e2e-tests/component/datasources/overview.ts
index f543311..54af58b 100644
--- a/web-console/e2e-tests/component/datasources/overview.ts
+++ b/web-console/e2e-tests/component/datasources/overview.ts
@@ -18,6 +18,8 @@
import * as playwright from 'playwright-core';
+import { clickButton } from '../../util/playwright';
+import { setLabeledInput } from '../../util/playwright';
import { extractTable } from '../../util/table';
import { CompactionConfig } from './compaction';
@@ -79,10 +81,14 @@ export class DatasourcesOverview {
await this.openEditActions(datasourceName);
await this.page.click('"Edit compaction configuration"');
- await this.setInput('Skip offset from latest', compactionConfig.skipOffsetFromLatest);
+ await setLabeledInput(
+ this.page,
+ 'Skip offset from latest',
+ compactionConfig.skipOffsetFromLatest,
+ );
await compactionConfig.partitionsSpec.apply(this.page);
- await this.clickButton('Submit');
+ await clickButton(this.page, 'Submit');
}
private async openEditActions(datasourceName: string): Promise<void> {
@@ -96,14 +102,4 @@ export class DatasourcesOverview {
editActions[index].click();
await this.page.waitFor(5000);
}
-
- private async setInput(label: string, value: string) {
- const input = await this.page.$(`//*[text()="${label}"]/following-sibling::div//input`);
- await input!.fill('');
- await input!.type(value);
- }
-
- private async clickButton(text: string) {
- await this.page.click(`//button/*[contains(text(),"${text}")]`, { waitUntil: 'load' } as any);
- }
}
diff --git a/web-console/e2e-tests/component/load-data/config/partition.ts b/web-console/e2e-tests/component/load-data/config/partition.ts
index 5ecba32..8431ecc 100644
--- a/web-console/e2e-tests/component/load-data/config/partition.ts
+++ b/web-console/e2e-tests/component/load-data/config/partition.ts
@@ -16,6 +16,13 @@
* limitations under the License.
*/
+import * as playwright from 'playwright-core';
+
+import { selectSuggestibleInput } from '../../../util/playwright';
+import { setLabeledInput } from '../../../util/playwright';
+
+/* tslint:disable max-classes-per-file */
+
/**
* Possible values for partition step segment granularity.
*/
@@ -26,17 +33,80 @@ export enum SegmentGranularity {
YEAR = 'YEAR',
}
+const PARTITIONING_TYPE = 'Partitioning type';
+
+export interface PartitionsSpec {
+ readonly type: string;
+ apply(page: playwright.Page): Promise<void>;
+}
+
+export class HashedPartitionsSpec implements PartitionsSpec {
+ readonly type: string;
+
+ constructor(props: HashedPartitionsSpecProps) {
+ Object.assign(this, props);
+ this.type = 'hashed';
+ }
+
+ async apply(page: playwright.Page): Promise<void> {
+ await setLabeledInput(page, PARTITIONING_TYPE, this.type);
+ if (this.numShards != null) {
+ await setLabeledInput(page, 'Num shards', String(this.numShards));
+ }
+ }
+}
+
+interface HashedPartitionsSpecProps {
+ readonly numShards: number | null;
+}
+
+export interface HashedPartitionsSpec extends HashedPartitionsSpecProps {}
+
+export class SingleDimPartitionsSpec implements PartitionsSpec {
+ readonly type: string;
+
+ constructor(props: SingleDimPartitionsSpecProps) {
+ Object.assign(this, props);
+ this.type = 'single_dim';
+ }
+
+ async apply(page: playwright.Page): Promise<void> {
+ await selectSuggestibleInput(page, PARTITIONING_TYPE, this.type);
+ await setLabeledInput(page, 'Partition dimension', this.partitionDimension);
+ if (this.targetRowsPerSegment) {
+ await setLabeledInput(page, 'Target rows per segment', String(this.targetRowsPerSegment));
+ }
+ if (this.maxRowsPerSegment) {
+ await setLabeledInput(page, 'Max rows per segment', String(this.maxRowsPerSegment));
+ }
+ }
+}
+
+interface SingleDimPartitionsSpecProps {
+ readonly partitionDimension: string;
+ readonly targetRowsPerSegment: number | null;
+ readonly maxRowsPerSegment: number | null;
+}
+
+export interface SingleDimPartitionsSpec extends SingleDimPartitionsSpecProps {}
+
/**
* Data loader partition step configuration.
*/
export class PartitionConfig {
+ readonly forceGuaranteedRollupText: string;
+
constructor(props: PartitionConfigProps) {
Object.assign(this, props);
+ this.forceGuaranteedRollupText = this.forceGuaranteedRollup ? 'True' : 'False';
}
}
interface PartitionConfigProps {
readonly segmentGranularity: SegmentGranularity;
+ readonly timeIntervals: string | null;
+ readonly forceGuaranteedRollup: boolean | null;
+ readonly partitionsSpec: PartitionsSpec | null;
}
export interface PartitionConfig extends PartitionConfigProps {}
diff --git a/web-console/e2e-tests/component/load-data/data-connector/data-connector.ts b/web-console/e2e-tests/component/load-data/data-connector/data-connector.ts
index 50a9d29..7112f26 100644
--- a/web-console/e2e-tests/component/load-data/data-connector/data-connector.ts
+++ b/web-console/e2e-tests/component/load-data/data-connector/data-connector.ts
@@ -16,10 +16,19 @@
* limitations under the License.
*/
+import * as playwright from 'playwright-core';
+
+import { clickButton } from '../../../util/playwright';
+
/**
* Connector for data loader input data.
*/
export interface DataConnector {
readonly name: string;
+ readonly needParse: boolean;
connect(): Promise<void>;
}
+
+export async function clickApplyButton(page: playwright.Page): Promise<void> {
+ await clickButton(page, 'Apply');
+}
diff --git a/web-console/e2e-tests/component/load-data/data-connector/local-file.ts b/web-console/e2e-tests/component/load-data/data-connector/local-file.ts
index 4e2aa36..dfbaed5 100644
--- a/web-console/e2e-tests/component/load-data/data-connector/local-file.ts
+++ b/web-console/e2e-tests/component/load-data/data-connector/local-file.ts
@@ -18,6 +18,9 @@
import * as playwright from 'playwright-core';
+import { setLabeledInput } from '../../../util/playwright';
+
+import { clickApplyButton } from './data-connector';
import { DataConnector } from './data-connector';
/**
@@ -25,30 +28,26 @@ import { DataConnector } from './data-connector';
*/
export class LocalFileDataConnector implements DataConnector {
readonly name: string;
+ readonly needParse: boolean;
private page: playwright.Page;
- private baseDirectory: string;
- private fileFilter: string;
- constructor(page: playwright.Page, baseDirectory: string, fileFilter: string) {
+ constructor(page: playwright.Page, props: LocalFileDataConnectorProps) {
+ Object.assign(this, props);
this.name = 'Local disk';
+ this.needParse = true;
this.page = page;
- this.baseDirectory = baseDirectory;
- this.fileFilter = fileFilter;
}
async connect() {
- const baseDirectoryInput = await this.page.$('input[placeholder="/path/to/files/"]');
- await this.setInput(baseDirectoryInput!, this.baseDirectory);
-
- const fileFilterInput = await this.page.$('input[value="*"]');
- await this.setInput(fileFilterInput!, this.fileFilter);
-
- const applyButton = await this.page.$('"Apply"');
- await applyButton!.click();
+ await setLabeledInput(this.page, 'Base directory', this.baseDirectory);
+ await setLabeledInput(this.page, 'File filter', this.fileFilter);
+ await clickApplyButton(this.page);
}
+}
- private async setInput(input: playwright.ElementHandle<Element>, value: string) {
- await input.fill('');
- await input.type(value);
- }
+interface LocalFileDataConnectorProps {
+ readonly baseDirectory: string;
+ readonly fileFilter: string;
}
+
+export interface LocalFileDataConnector extends LocalFileDataConnectorProps {}
diff --git a/web-console/e2e-tests/component/load-data/data-connector/local-file.ts b/web-console/e2e-tests/component/load-data/data-connector/reindex.ts
similarity index 54%
copy from web-console/e2e-tests/component/load-data/data-connector/local-file.ts
copy to web-console/e2e-tests/component/load-data/data-connector/reindex.ts
index 4e2aa36..d519a5d 100644
--- a/web-console/e2e-tests/component/load-data/data-connector/local-file.ts
+++ b/web-console/e2e-tests/component/load-data/data-connector/reindex.ts
@@ -18,37 +18,36 @@
import * as playwright from 'playwright-core';
+import { setLabeledInput } from '../../../util/playwright';
+
+import { clickApplyButton } from './data-connector';
import { DataConnector } from './data-connector';
/**
- * Local file connector for data loader input data.
+ * Reindexing connector for data loader input data.
*/
-export class LocalFileDataConnector implements DataConnector {
+export class ReindexDataConnector implements DataConnector {
readonly name: string;
+ readonly needParse: boolean;
private page: playwright.Page;
- private baseDirectory: string;
- private fileFilter: string;
- constructor(page: playwright.Page, baseDirectory: string, fileFilter: string) {
- this.name = 'Local disk';
+ constructor(page: playwright.Page, props: ReindexDataConnectorProps) {
+ Object.assign(this, props);
+ this.name = 'Reindex from Druid';
+ this.needParse = false;
this.page = page;
- this.baseDirectory = baseDirectory;
- this.fileFilter = fileFilter;
}
async connect() {
- const baseDirectoryInput = await this.page.$('input[placeholder="/path/to/files/"]');
- await this.setInput(baseDirectoryInput!, this.baseDirectory);
-
- const fileFilterInput = await this.page.$('input[value="*"]');
- await this.setInput(fileFilterInput!, this.fileFilter);
-
- const applyButton = await this.page.$('"Apply"');
- await applyButton!.click();
+ await setLabeledInput(this.page, 'Datasource', this.datasourceName);
+ await setLabeledInput(this.page, 'Interval', this.interval);
+ await clickApplyButton(this.page);
}
+}
- private async setInput(input: playwright.ElementHandle<Element>, value: string) {
- await input.fill('');
- await input.type(value);
- }
+interface ReindexDataConnectorProps {
+ readonly datasourceName: string;
+ readonly interval: string;
}
+
+export interface ReindexDataConnector extends ReindexDataConnectorProps {}
diff --git a/web-console/e2e-tests/component/load-data/data-loader.ts b/web-console/e2e-tests/component/load-data/data-loader.ts
index 0244c43..4d3c971 100644
--- a/web-console/e2e-tests/component/load-data/data-loader.ts
+++ b/web-console/e2e-tests/component/load-data/data-loader.ts
@@ -18,6 +18,11 @@
import * as playwright from 'playwright-core';
+import { clickButton } from '../../util/playwright';
+import { clickLabeledButton } from '../../util/playwright';
+import { setLabeledInput } from '../../util/playwright';
+import { setLabeledTextarea } from '../../util/playwright';
+
import { ConfigureSchemaConfig } from './config/configure-schema';
import { PartitionConfig } from './config/partition';
import { PublishConfig } from './config/publish';
@@ -41,8 +46,10 @@ export class DataLoader {
await this.page.goto(this.baseUrl);
await this.start();
await this.connect(this.connector, this.connectValidator);
- await this.parseData();
- await this.parseTime();
+ if (this.connector.needParse) {
+ await this.parseData();
+ await this.parseTime();
+ }
await this.transform();
await this.filter();
await this.configureSchema(this.configureSchemaConfig);
@@ -54,13 +61,14 @@ export class DataLoader {
private async start() {
await this.page.click(`"${this.connector.name}"`);
- await this.clickButton('Connect data');
+ await clickButton(this.page, 'Connect data');
}
private async connect(connector: DataConnector, validator: (preview: string) => void) {
await connector.connect();
await this.validateConnect(validator);
- await this.clickButton('Next: Parse data');
+ const next = this.connector.needParse ? 'Parse data' : 'Transform';
+ await clickButton(this.page, `Next: ${next}`);
}
private async validateConnect(validator: (preview: string) => void) {
@@ -72,28 +80,28 @@ export class DataLoader {
private async parseData() {
await this.page.waitFor('.parse-data-table');
- await this.clickButton('Next: Parse time');
+ await clickButton(this.page, 'Next: Parse time');
}
private async parseTime() {
await this.page.waitFor('.parse-time-table');
- await this.clickButton('Next: Transform');
+ await clickButton(this.page, 'Next: Transform');
}
private async transform() {
await this.page.waitFor('.transform-table');
- await this.clickButton('Next: Filter');
+ await clickButton(this.page, 'Next: Filter');
}
private async filter() {
await this.page.waitFor('.filter-table');
- await this.clickButton('Next: Configure schema');
+ await clickButton(this.page, 'Next: Configure schema');
}
private async configureSchema(configureSchemaConfig: ConfigureSchemaConfig) {
await this.page.waitFor('.schema-table');
await this.applyConfigureSchemaConfig(configureSchemaConfig);
- await this.clickButton('Next: Partition');
+ await clickButton(this.page, 'Next: Partition');
}
private async applyConfigureSchemaConfig(configureSchemaConfig: ConfigureSchemaConfig) {
@@ -103,7 +111,7 @@ export class DataLoader {
await rollup!.click();
const confirmationDialogSelector = '//*[contains(@class,"bp3-alert-body")]';
await this.page.waitFor(confirmationDialogSelector);
- await this.clickButton('Yes');
+ await clickButton(this.page, 'Yes');
const statusMessageSelector = '.recipe-toaster';
await this.page.waitFor(statusMessageSelector);
await this.page.click(`${statusMessageSelector} button`);
@@ -113,48 +121,44 @@ export class DataLoader {
private async partition(partitionConfig: PartitionConfig) {
await this.page.waitFor('div.load-data-view.partition');
await this.applyPartitionConfig(partitionConfig);
- await this.clickButton('Next: Tune');
+ await clickButton(this.page, 'Next: Tune');
}
private async applyPartitionConfig(partitionConfig: PartitionConfig) {
- const segmentGranularity = await this.page.$(
- '//*[text()="Segment granularity"]/following-sibling::div//input',
- );
- await this.setInput(segmentGranularity!, partitionConfig.segmentGranularity);
+ await setLabeledInput(this.page, 'Segment granularity', partitionConfig.segmentGranularity);
+ if (partitionConfig.forceGuaranteedRollup) {
+ await clickLabeledButton(
+ this.page,
+ 'Force guaranteed rollup',
+ partitionConfig.forceGuaranteedRollupText,
+ );
+ await setLabeledTextarea(this.page, 'Time intervals', partitionConfig.timeIntervals!);
+ }
+ if (partitionConfig.partitionsSpec != null) {
+ await partitionConfig.partitionsSpec.apply(this.page);
+ }
}
private async tune() {
await this.page.waitFor('div.load-data-view.tuning');
- await this.clickButton('Next: Publish');
+ await clickButton(this.page, 'Next: Publish');
}
private async publish(publishConfig: PublishConfig) {
await this.page.waitFor('div.load-data-view.publish');
await this.applyPublishConfig(publishConfig);
- await this.clickButton('Edit spec');
+ await clickButton(this.page, 'Edit spec');
}
private async applyPublishConfig(publishConfig: PublishConfig) {
if (publishConfig.datasourceName != null) {
- const datasourceName = await this.page.$(
- '//*[text()="Datasource name"]/following-sibling::div//input',
- );
- await this.setInput(datasourceName!, publishConfig.datasourceName);
+ await setLabeledInput(this.page, 'Datasource name', publishConfig.datasourceName);
}
}
private async editSpec() {
await this.page.waitFor('div.load-data-view.spec');
- await this.clickButton('Submit');
- }
-
- private async clickButton(text: string) {
- await this.page.click(`//button/*[contains(text(),"${text}")]`, { waitUntil: 'load' } as any);
- }
-
- private async setInput(input: playwright.ElementHandle<Element>, value: string) {
- await input.fill('');
- await input.type(value);
+ await clickButton(this.page, 'Submit');
}
}
diff --git a/web-console/e2e-tests/component/query/overview.ts b/web-console/e2e-tests/component/query/overview.ts
index 973d29a..3175a6d 100644
--- a/web-console/e2e-tests/component/query/overview.ts
+++ b/web-console/e2e-tests/component/query/overview.ts
@@ -18,6 +18,8 @@
import * as playwright from 'playwright-core';
+import { clickButton } from '../../util/playwright';
+import { setInput } from '../../util/playwright';
import { extractTable } from '../../util/table';
/**
@@ -37,19 +39,10 @@ export class QueryOverview {
await this.page.reload({ waitUntil: 'networkidle0' });
const input = await this.page.$('div.query-input textarea');
- await this.setInput(input!, query);
- await this.clickButton('Run');
+ await setInput(input!, query);
+ await clickButton(this.page, 'Run');
await this.page.waitFor('div.query-info');
return await extractTable(this.page, 'div.query-output div.rt-tr-group', 'div.rt-td');
}
-
- private async setInput(input: playwright.ElementHandle<Element>, value: string) {
- await input.fill('');
- await input.type(value);
- }
-
- private async clickButton(text: string) {
- await this.page.click(`//button/*[contains(text(),"${text}")]`, { waitUntil: 'load' } as any);
- }
}
diff --git a/web-console/e2e-tests/tutorial-batch.spec.ts b/web-console/e2e-tests/reindexing.spec.ts
similarity index 63%
copy from web-console/e2e-tests/tutorial-batch.spec.ts
copy to web-console/e2e-tests/reindexing.spec.ts
index 5908938..b735578 100644
--- a/web-console/e2e-tests/tutorial-batch.spec.ts
+++ b/web-console/e2e-tests/reindexing.spec.ts
@@ -16,20 +16,22 @@
* limitations under the License.
*/
+import path from 'path';
import * as playwright from 'playwright-core';
-import { v4 as uuid } from 'uuid';
import { DatasourcesOverview } from './component/datasources/overview';
import { IngestionOverview } from './component/ingestion/overview';
import { ConfigureSchemaConfig } from './component/load-data/config/configure-schema';
import { PartitionConfig } from './component/load-data/config/partition';
import { SegmentGranularity } from './component/load-data/config/partition';
+import { SingleDimPartitionsSpec } from './component/load-data/config/partition';
import { PublishConfig } from './component/load-data/config/publish';
-import { LocalFileDataConnector } from './component/load-data/data-connector/local-file';
+import { ReindexDataConnector } from './component/load-data/data-connector/reindex';
import { DataLoader } from './component/load-data/data-loader';
-import { QueryOverview } from './component/query/overview';
import { saveScreenshotIfError } from './util/debug';
+import { DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR } from './util/druid';
import { UNIFIED_CONSOLE_URL } from './util/druid';
+import { runIndexTask } from './util/druid';
import { createBrowserNormal as createBrowser } from './util/playwright';
import { createPage } from './util/playwright';
import { retryIfJestAssertionError } from './util/retry';
@@ -37,7 +39,7 @@ import { waitTillWebConsoleReady } from './util/setup';
jest.setTimeout(5 * 60 * 1000);
-describe('Tutorial: Loading a file', () => {
+describe('Reindexing from Druid', () => {
let browser: playwright.Browser;
let page: playwright.Page;
@@ -54,15 +56,25 @@ describe('Tutorial: Loading a file', () => {
await browser.close();
});
- it('Loads data from local disk', async () => {
- const datasourceName = uuid();
- const dataConnector = new LocalFileDataConnector(
- page,
- 'quickstart/tutorial/',
- 'wikiticker-2015-09-12-sampled.json.gz',
- );
+ it('Reindex datasource from dynamic to single dim partitions', async () => {
+ const testName = 'reindex-dynamic-to-single-dim-';
+ const datasourceName = testName + new Date().toISOString();
+ const interval = '2015-09-12/2015-09-13';
+ const dataConnector = new ReindexDataConnector(page, {
+ datasourceName,
+ interval,
+ });
const configureSchemaConfig = new ConfigureSchemaConfig({ rollup: false });
- const partitionConfig = new PartitionConfig({ segmentGranularity: SegmentGranularity.DAY });
+ const partitionConfig = new PartitionConfig({
+ segmentGranularity: SegmentGranularity.DAY,
+ timeIntervals: interval,
+ forceGuaranteedRollup: true,
+ partitionsSpec: new SingleDimPartitionsSpec({
+ partitionDimension: 'channel',
+ targetRowsPerSegment: 10_000,
+ maxRowsPerSegment: null,
+ }),
+ });
const publishConfig = new PublishConfig({ datasourceName: datasourceName });
const dataLoader = new DataLoader({
@@ -75,66 +87,79 @@ describe('Tutorial: Loading a file', () => {
publishConfig: publishConfig,
});
- await saveScreenshotIfError('load-data-from-local-disk-', page, async () => {
+ loadInitialData(datasourceName);
+
+ await saveScreenshotIfError(testName, page, async () => {
+ const numInitialSegment = 1;
+ await validateDatasourceStatus(page, datasourceName, numInitialSegment);
+
await dataLoader.load();
await validateTaskStatus(page, datasourceName);
- await validateDatasourceStatus(page, datasourceName);
- await validateQuery(page, datasourceName);
+
+ const numReindexedSegment = 4; // 39k rows into segments of ~10k rows
+ await validateDatasourceStatus(page, datasourceName, numReindexedSegment);
});
});
});
+function loadInitialData(datasourceName: string) {
+ const ingestionSpec = path.join(DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR, 'wikipedia-index.json');
+ const setDatasourceName = `s/wikipedia/${datasourceName}/`;
+ const sedCommands = [setDatasourceName];
+ runIndexTask(ingestionSpec, sedCommands);
+}
+
function validateConnectLocalData(preview: string) {
const lines = preview.split('\n');
expect(lines.length).toBe(500);
const firstLine = lines[0];
expect(firstLine).toBe(
- '{' +
- '"time":"2015-09-12T00:46:58.771Z"' +
+ 'Druid row: {' +
+ '"__time":1442018818771' +
+ ',"isRobot":"false"' +
+ ',"countryIsoCode":null' +
+ ',"added":"36"' +
+ ',"regionName":null' +
',"channel":"#en.wikipedia"' +
+ ',"delta":"36"' +
+ ',"isUnpatrolled":"false"' +
+ ',"isNew":"false"' +
+ ',"isMinor":"false"' +
+ ',"isAnonymous":"false"' +
+ ',"deleted":"0"' +
',"cityName":null' +
- ',"comment":"added project"' +
- ',"countryIsoCode":null' +
- ',"countryName":null' +
- ',"isAnonymous":false' +
- ',"isMinor":false' +
- ',"isNew":false' +
- ',"isRobot":false' +
- ',"isUnpatrolled":false' +
',"metroCode":null' +
',"namespace":"Talk"' +
+ ',"comment":"added project"' +
+ ',"countryName":null' +
',"page":"Talk:Oswald Tilghman"' +
- ',"regionIsoCode":null' +
- ',"regionName":null' +
',"user":"GELongstreet"' +
- ',"delta":36' +
- ',"added":36' +
- ',"deleted":0' +
+ ',"regionIsoCode":null' +
'}',
);
const lastLine = lines[lines.length - 1];
expect(lastLine).toBe(
- '{' +
- '"time":"2015-09-12T01:11:54.823Z"' +
+ 'Druid row: {' +
+ '"__time":1442020314823' +
+ ',"isRobot":"false"' +
+ ',"countryIsoCode":null' +
+ ',"added":"1"' +
+ ',"regionName":null' +
',"channel":"#en.wikipedia"' +
+ ',"delta":"1"' +
+ ',"isUnpatrolled":"false"' +
+ ',"isNew":"false"' +
+ ',"isMinor":"true"' +
+ ',"isAnonymous":"false"' +
+ ',"deleted":"0"' +
',"cityName":null' +
- ',"comment":"/* History */[[WP:AWB/T|Typo fixing]], [[WP:AWB/T|typo(s) fixed]]: nothern → northern using [[Project:AWB|AWB]]"' +
- ',"countryIsoCode":null' +
- ',"countryName":null' +
- ',"isAnonymous":false' +
- ',"isMinor":true' +
- ',"isNew":false' +
- ',"isRobot":false' +
- ',"isUnpatrolled":false' +
',"metroCode":null' +
',"namespace":"Main"' +
+ ',"comment":"/* History */[[WP:AWB/T|Typo fixing]], [[WP:AWB/T|typo(s) fixed]]: nothern → northern using [[Project:AWB|AWB]]"' +
+ ',"countryName":null' +
',"page":"Hapoel Katamon Jerusalem F.C."' +
- ',"regionIsoCode":null' +
- ',"regionName":null' +
',"user":"The Quixotic Potato"' +
- ',"delta":1' +
- ',"added":1' +
- ',"deleted":0' +
+ ',"regionIsoCode":null' +
'}',
);
}
@@ -150,44 +175,19 @@ async function validateTaskStatus(page: playwright.Page, datasourceName: string)
});
}
-async function validateDatasourceStatus(page: playwright.Page, datasourceName: string) {
+async function validateDatasourceStatus(
+ page: playwright.Page,
+ datasourceName: string,
+ expectedNumSegment: number,
+) {
const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL);
+ const numSegmentString = `${expectedNumSegment} segment` + (expectedNumSegment !== 1 ? 's' : '');
await retryIfJestAssertionError(async () => {
const datasources = await datasourcesOverview.getDatasources();
const datasource = datasources.find(t => t.name === datasourceName);
expect(datasource).toBeDefined();
- expect(datasource!.availability).toMatch('Fully available (1 segment)');
+ expect(datasource!.availability).toMatch(`Fully available (${numSegmentString})`);
expect(datasource!.totalRows).toBe(39244);
});
}
-
-async function validateQuery(page: playwright.Page, datasourceName: string) {
- const queryOverview = new QueryOverview(page, UNIFIED_CONSOLE_URL);
- const query = `SELECT * FROM "${datasourceName}" ORDER BY __time`;
- const results = await queryOverview.runQuery(query);
- expect(results).toBeDefined();
- expect(results.length).toBeGreaterThan(0);
- expect(results[0]).toStrictEqual([
- /* __time */ '2015-09-12T00:46:58.771Z',
- /* added */ '36',
- /* channel */ '#en.wikipedia',
- /* cityName */ 'null',
- /* comment */ 'added project',
- /* countryIsoCode */ 'null',
- /* countryName */ 'null',
- /* deleted */ '0',
- /* delta */ '36',
- /* isAnonymous */ 'false',
- /* isMinor */ 'false',
- /* isNew */ 'false',
- /* isRobot */ 'false',
- /* isUnpatrolled */ 'false',
- /* metroCode */ 'null',
- /* namespace */ 'Talk',
- /* page */ 'Talk:Oswald Tilghman',
- /* regionIsoCode */ 'null',
- /* regionName */ 'null',
- /* user */ 'GELongstreet',
- ]);
-}
diff --git a/web-console/e2e-tests/tutorial-batch.spec.ts b/web-console/e2e-tests/tutorial-batch.spec.ts
index 5908938..c655663 100644
--- a/web-console/e2e-tests/tutorial-batch.spec.ts
+++ b/web-console/e2e-tests/tutorial-batch.spec.ts
@@ -17,7 +17,6 @@
*/
import * as playwright from 'playwright-core';
-import { v4 as uuid } from 'uuid';
import { DatasourcesOverview } from './component/datasources/overview';
import { IngestionOverview } from './component/ingestion/overview';
@@ -55,14 +54,19 @@ describe('Tutorial: Loading a file', () => {
});
it('Loads data from local disk', async () => {
- const datasourceName = uuid();
- const dataConnector = new LocalFileDataConnector(
- page,
- 'quickstart/tutorial/',
- 'wikiticker-2015-09-12-sampled.json.gz',
- );
+ const testName = 'load-data-from-local-disk-';
+ const datasourceName = testName + new Date().toISOString();
+ const dataConnector = new LocalFileDataConnector(page, {
+ baseDirectory: 'quickstart/tutorial/',
+ fileFilter: 'wikiticker-2015-09-12-sampled.json.gz',
+ });
const configureSchemaConfig = new ConfigureSchemaConfig({ rollup: false });
- const partitionConfig = new PartitionConfig({ segmentGranularity: SegmentGranularity.DAY });
+ const partitionConfig = new PartitionConfig({
+ segmentGranularity: SegmentGranularity.DAY,
+ timeIntervals: null,
+ forceGuaranteedRollup: null,
+ partitionsSpec: null,
+ });
const publishConfig = new PublishConfig({ datasourceName: datasourceName });
const dataLoader = new DataLoader({
@@ -75,7 +79,7 @@ describe('Tutorial: Loading a file', () => {
publishConfig: publishConfig,
});
- await saveScreenshotIfError('load-data-from-local-disk-', page, async () => {
+ await saveScreenshotIfError(testName, page, async () => {
await dataLoader.load();
await validateTaskStatus(page, datasourceName);
await validateDatasourceStatus(page, datasourceName);
diff --git a/web-console/e2e-tests/util/druid.ts b/web-console/e2e-tests/util/druid.ts
index 617be85..87b5d41 100644
--- a/web-console/e2e-tests/util/druid.ts
+++ b/web-console/e2e-tests/util/druid.ts
@@ -16,6 +16,7 @@
* limitations under the License.
*/
+import { execSync } from 'child_process';
import path from 'path';
export const UNIFIED_CONSOLE_URL = 'http://localhost:8888/unified-console.html';
@@ -24,4 +25,24 @@ export const COORDINATOR_URL = 'http://localhost:8081';
const UTIL_DIR = __dirname;
const E2E_TEST_DIR = path.dirname(UTIL_DIR);
const WEB_CONSOLE_DIR = path.dirname(E2E_TEST_DIR);
-export const DRUID_DIR = path.dirname(WEB_CONSOLE_DIR);
+const DRUID_DIR = path.dirname(WEB_CONSOLE_DIR);
+export const DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR = path.join(
+ DRUID_DIR,
+ 'examples',
+ 'quickstart',
+ 'tutorial',
+);
+
+export function runIndexTask(ingestionSpecPath: string, sedCommands: Array<string>) {
+ const postIndexTask = path.join(DRUID_DIR, 'examples', 'bin', 'post-index-task');
+ const sedCommandsString = sedCommands.map(sedCommand => `-e '${sedCommand}'`).join(' ');
+ execSync(
+ `${postIndexTask} \
+ --file <(sed ${sedCommandsString} ${ingestionSpecPath}) \
+ --url ${COORDINATOR_URL}`,
+ {
+ shell: 'bash',
+ timeout: 3 * 60 * 1000,
+ },
+ );
+}
diff --git a/web-console/e2e-tests/util/playwright.ts b/web-console/e2e-tests/util/playwright.ts
index 21cbd33..e31a31b 100644
--- a/web-console/e2e-tests/util/playwright.ts
+++ b/web-console/e2e-tests/util/playwright.ts
@@ -48,3 +48,62 @@ export async function createPage(browser: playwright.Browser): Promise<playwrigh
await page.setViewportSize({ width: WIDTH, height: HEIGHT });
return page;
}
+
+export async function setLabeledInput(
+ page: playwright.Page,
+ label: string,
+ value: string,
+): Promise<void> {
+ return setLabeledElement(page, 'input', label, value);
+}
+
+export async function setLabeledTextarea(
+ page: playwright.Page,
+ label: string,
+ value: string,
+): Promise<void> {
+ return setLabeledElement(page, 'textarea', label, value);
+}
+
+async function setLabeledElement(
+ page: playwright.Page,
+ type: string,
+ label: string,
+ value: string,
+): Promise<void> {
+ const element = await page.$(`//*[text()="${label}"]/following-sibling::div//${type}`);
+ await setInput(element!, value);
+}
+
+export async function setInput(
+ input: playwright.ElementHandle<Element>,
+ value: string,
+): Promise<void> {
+ await input.fill('');
+ await input.type(value);
+}
+
+function buttonSelector(text: string) {
+ return `//button/*[contains(text(),"${text}")]`;
+}
+
+export async function clickButton(page: playwright.Page, text: string): Promise<void> {
+ await page.click(buttonSelector(text));
+}
+
+export async function clickLabeledButton(
+ page: playwright.Page,
+ label: string,
+ text: string,
+): Promise<void> {
+ await page.click(`//*[text()="${label}"]/following-sibling::div${buttonSelector(text)}`);
+}
+
+export async function selectSuggestibleInput(
+ page: playwright.Page,
+ label: string,
+ value: string,
+): Promise<void> {
+ await page.click(`//*[text()="${label}"]/following-sibling::div//button`);
+ await page.click(`"${value}"`);
+}
diff --git a/web-console/package.json b/web-console/package.json
index bce8dc5..6d132e9 100644
--- a/web-console/package.json
+++ b/web-console/package.json
@@ -39,7 +39,7 @@
"test-base": "npm run tslint && npm run sasslint && npm run jest",
"test": "npm run test-base -- --silent 2>&1",
"test-ci": "npm run test-base -- --coverage",
- "test-e2e": "jest --config jest.e2e.config.js --detectOpenHandles e2e-tests",
+ "test-e2e": "jest --config jest.e2e.config.js e2e-tests",
"codecov": "codecov --disable=gcov -p ..",
"coverage": "jest --coverage src",
"update-snapshots": "jest -u",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org