You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by vo...@apache.org on 2020/09/26 01:29:36 UTC
[druid] branch master updated: Web console autocompaction E2E test
(#10425)
This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new cbd9ac8 Web console autocompaction E2E test (#10425)
cbd9ac8 is described below
commit cbd9ac8592f4e7cc8eb8ace4cfa42355cf0fffb6
Author: Chi Cao Minh <ch...@imply.io>
AuthorDate: Fri Sep 25 18:28:25 2020 -0700
Web console autocompaction E2E test (#10425)
Add an E2E test for the common case web console workflow of setting up
autocompaction that changes the partitions from dynamic to hashed.
Also fix an issue with the async test setup to properly wait for the web
console to be ready.
---
web-console/e2e-tests/auto-compaction.spec.ts | 164 +++++++++++++++++++++
.../e2e-tests/component/datasources/compaction.ts | 70 +++++++++
.../e2e-tests/component/datasources/overview.ts | 36 +++++
web-console/e2e-tests/tutorial-batch.spec.ts | 2 +
web-console/e2e-tests/util/druid.ts | 8 +
web-console/e2e-tests/util/setup.ts | 10 +-
web-console/jest.e2e.config.js | 5 +-
7 files changed, 287 insertions(+), 8 deletions(-)
diff --git a/web-console/e2e-tests/auto-compaction.spec.ts b/web-console/e2e-tests/auto-compaction.spec.ts
new file mode 100644
index 0000000..e608d1c
--- /dev/null
+++ b/web-console/e2e-tests/auto-compaction.spec.ts
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import axios from 'axios';
+import { execSync } from 'child_process';
+import path from 'path';
+import * as playwright from 'playwright-core';
+import { v4 as uuid } from 'uuid';
+
+import { CompactionConfig } from './component/datasources/compaction';
+import { CompactionHashPartitionsSpec } from './component/datasources/compaction';
+import { Datasource } from './component/datasources/datasource';
+import { DatasourcesOverview } from './component/datasources/overview';
+import { saveScreenshotIfError } from './util/debug';
+import { COORDINATOR_URL } from './util/druid';
+import { DRUID_DIR } from './util/druid';
+import { UNIFIED_CONSOLE_URL } from './util/druid';
+import { createBrowserNormal as createBrowser } from './util/playwright';
+import { createPage } from './util/playwright';
+import { retryIfJestAssertionError } from './util/retry';
+import { waitTillWebConsoleReady } from './util/setup';
+
+jest.setTimeout(5 * 60 * 1000);
+
+// The workflow in these tests is based on the compaction tutorial:
+// https://druid.apache.org/docs/latest/tutorials/tutorial-compaction.html
+describe('Auto-compaction', () => {
+ let browser: playwright.Browser;
+ let page: playwright.Page;
+
+ beforeAll(async () => {
+ await waitTillWebConsoleReady();
+ browser = await createBrowser();
+ });
+
+ beforeEach(async () => {
+ page = await createPage(browser);
+ });
+
+ afterAll(async () => {
+ await browser.close();
+ });
+
+ it('Compacts segments from dynamic to hash partitions', async () => {
+ const datasourceName = uuid();
+ loadInitialData(datasourceName);
+
+ await saveScreenshotIfError('auto-compaction-', page, async () => {
+ const uncompactedNumSegment = 3;
+ const numRow = 1412;
+ await validateDatasourceStatus(page, datasourceName, uncompactedNumSegment, numRow);
+
+ const compactionConfig = new CompactionConfig({
+ skipOffsetFromLatest: 'PT0S',
+ partitionsSpec: new CompactionHashPartitionsSpec({
+ numShards: 1,
+ }),
+ });
+ await configureCompaction(page, datasourceName, compactionConfig);
+
+ // Depending on the number of configured tasks slots, autocompaction may
+ // need several iterations if several time chunks need compaction
+ let currNumSegment = uncompactedNumSegment;
+ await retryIfJestAssertionError(async () => {
+ await triggerCompaction();
+ currNumSegment = await waitForCompaction(page, datasourceName, currNumSegment);
+
+ const compactedNumSegment = 2;
+ expect(currNumSegment).toBe(compactedNumSegment);
+ });
+ });
+ });
+});
+
+function loadInitialData(datasourceName: string) {
+ const postIndexTask = path.join(DRUID_DIR, 'examples', 'bin', 'post-index-task');
+ const ingestionSpec = path.join(
+ DRUID_DIR,
+ 'examples',
+ 'quickstart',
+ 'tutorial',
+ 'compaction-init-index.json',
+ );
+ const setDatasourceName = `s/compaction-tutorial/${datasourceName}/`;
+ const setIntervals = 's|2015-09-12/2015-09-13|2015-09-12/2015-09-12T02:00|'; // shorten to reduce test duration
+ execSync(
+ `${postIndexTask} \
+ --file <(sed -e '${setDatasourceName}' -e '${setIntervals}' ${ingestionSpec}) \
+ --url ${COORDINATOR_URL}`,
+ {
+ shell: 'bash',
+ timeout: 3 * 60 * 1000,
+ },
+ );
+}
+
+async function validateDatasourceStatus(
+ page: playwright.Page,
+ datasourceName: string,
+ expectedNumSegment: number,
+ expectedNumRow: number,
+) {
+ await retryIfJestAssertionError(async () => {
+ const datasource = await getDatasource(page, datasourceName);
+ expect(datasource.availability).toMatch(`Fully available (${expectedNumSegment} segments)`);
+ expect(datasource.totalRows).toBe(expectedNumRow);
+ });
+}
+
+async function getDatasource(page: playwright.Page, datasourceName: string): Promise<Datasource> {
+ const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL);
+ const datasources = await datasourcesOverview.getDatasources();
+ const datasource = datasources.find(t => t.name === datasourceName);
+ expect(datasource).toBeDefined();
+ return datasource!;
+}
+
+async function configureCompaction(
+ page: playwright.Page,
+ datasourceName: string,
+ compactionConfig: CompactionConfig,
+) {
+ const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL);
+ await datasourcesOverview.setCompactionConfiguration(datasourceName, compactionConfig);
+}
+
+async function triggerCompaction() {
+ const res = await axios.post(`${COORDINATOR_URL}/druid/coordinator/v1/compaction/compact`);
+ expect(res.status).toBe(200);
+}
+
+async function waitForCompaction(
+ page: playwright.Page,
+ datasourceName: string,
+ prevNumSegment: number,
+): Promise<number> {
+ await retryIfJestAssertionError(async () => {
+ const currNumSegment = await getNumSegment(page, datasourceName);
+ expect(currNumSegment).toBeLessThan(prevNumSegment);
+ });
+
+ return getNumSegment(page, datasourceName);
+}
+
+async function getNumSegment(page: playwright.Page, datasourceName: string): Promise<number> {
+ const datasource = await getDatasource(page, datasourceName);
+ const currNumSegmentString = datasource!.availability.match(/(\d+)/)![0];
+ return Number(currNumSegmentString);
+}
diff --git a/web-console/e2e-tests/component/datasources/compaction.ts b/web-console/e2e-tests/component/datasources/compaction.ts
new file mode 100644
index 0000000..6aa1a95
--- /dev/null
+++ b/web-console/e2e-tests/component/datasources/compaction.ts
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as playwright from 'playwright-core';
+
+/* tslint:disable max-classes-per-file */
+
+const PARTITIONING_TYPE = 'Partitioning type';
+
+interface CompactionPartitionsSpec {
+ readonly type: string;
+ apply(page: playwright.Page): Promise<void>;
+}
+
+export class CompactionHashPartitionsSpec implements CompactionPartitionsSpec {
+ readonly type: string;
+
+ constructor(props: CompactionHashPartitionsSpecProps) {
+ Object.assign(this, props);
+ this.type = 'hashed';
+ }
+
+ async apply(page: playwright.Page): Promise<void> {
+ await setInput(page, PARTITIONING_TYPE, this.type);
+ await setInput(page, 'Num shards', String(this.numShards));
+ }
+}
+
+async function setInput(page: playwright.Page, label: string, value: string): Promise<void> {
+ const input = await page.$(`//*[text()="${label}"]/following-sibling::div//input`);
+ await input!.fill('');
+ await input!.type(value);
+}
+
+interface CompactionHashPartitionsSpecProps {
+ readonly numShards: number;
+}
+
+export interface CompactionHashPartitionsSpec extends CompactionHashPartitionsSpecProps {}
+
+/**
+ * Datasource compaction configuration
+ */
+export class CompactionConfig {
+ constructor(props: CompactionConfigProps) {
+ Object.assign(this, props);
+ }
+}
+
+interface CompactionConfigProps {
+ readonly skipOffsetFromLatest: string;
+ readonly partitionsSpec: CompactionPartitionsSpec;
+}
+
+export interface CompactionConfig extends CompactionConfigProps {}
diff --git a/web-console/e2e-tests/component/datasources/overview.ts b/web-console/e2e-tests/component/datasources/overview.ts
index 5b55eed..f543311 100644
--- a/web-console/e2e-tests/component/datasources/overview.ts
+++ b/web-console/e2e-tests/component/datasources/overview.ts
@@ -20,6 +20,7 @@ import * as playwright from 'playwright-core';
import { extractTable } from '../../util/table';
+import { CompactionConfig } from './compaction';
import { Datasource } from './datasource';
/**
@@ -70,4 +71,39 @@ export class DatasourcesOverview {
private static parseNumber(text: string): number {
return Number(text.replace(/,/g, ''));
}
+
+ async setCompactionConfiguration(
+ datasourceName: string,
+ compactionConfig: CompactionConfig,
+ ): Promise<void> {
+ await this.openEditActions(datasourceName);
+
+ await this.page.click('"Edit compaction configuration"');
+ await this.setInput('Skip offset from latest', compactionConfig.skipOffsetFromLatest);
+ await compactionConfig.partitionsSpec.apply(this.page);
+
+ await this.clickButton('Submit');
+ }
+
+ private async openEditActions(datasourceName: string): Promise<void> {
+ const datasources = await this.getDatasources();
+ const index = datasources.findIndex(t => t.name === datasourceName);
+ if (index < 0) {
+ throw new Error(`Could not find datasource: ${datasourceName}`);
+ }
+
+ const editActions = await this.page.$$('span[icon=wrench]');
+ editActions[index].click();
+ await this.page.waitFor(5000);
+ }
+
+ private async setInput(label: string, value: string) {
+ const input = await this.page.$(`//*[text()="${label}"]/following-sibling::div//input`);
+ await input!.fill('');
+ await input!.type(value);
+ }
+
+ private async clickButton(text: string) {
+ await this.page.click(`//button/*[contains(text(),"${text}")]`, { waitUntil: 'load' } as any);
+ }
}
diff --git a/web-console/e2e-tests/tutorial-batch.spec.ts b/web-console/e2e-tests/tutorial-batch.spec.ts
index 52aed72..5908938 100644
--- a/web-console/e2e-tests/tutorial-batch.spec.ts
+++ b/web-console/e2e-tests/tutorial-batch.spec.ts
@@ -33,6 +33,7 @@ import { UNIFIED_CONSOLE_URL } from './util/druid';
import { createBrowserNormal as createBrowser } from './util/playwright';
import { createPage } from './util/playwright';
import { retryIfJestAssertionError } from './util/retry';
+import { waitTillWebConsoleReady } from './util/setup';
jest.setTimeout(5 * 60 * 1000);
@@ -41,6 +42,7 @@ describe('Tutorial: Loading a file', () => {
let page: playwright.Page;
beforeAll(async () => {
+ await waitTillWebConsoleReady();
browser = await createBrowser();
});
diff --git a/web-console/e2e-tests/util/druid.ts b/web-console/e2e-tests/util/druid.ts
index 6ad9f63..617be85 100644
--- a/web-console/e2e-tests/util/druid.ts
+++ b/web-console/e2e-tests/util/druid.ts
@@ -16,4 +16,12 @@
* limitations under the License.
*/
+import path from 'path';
+
export const UNIFIED_CONSOLE_URL = 'http://localhost:8888/unified-console.html';
+export const COORDINATOR_URL = 'http://localhost:8081';
+
+const UTIL_DIR = __dirname;
+const E2E_TEST_DIR = path.dirname(UTIL_DIR);
+const WEB_CONSOLE_DIR = path.dirname(E2E_TEST_DIR);
+export const DRUID_DIR = path.dirname(WEB_CONSOLE_DIR);
diff --git a/web-console/e2e-tests/util/setup.ts b/web-console/e2e-tests/util/setup.ts
index 8c306cc..081f9ba 100644
--- a/web-console/e2e-tests/util/setup.ts
+++ b/web-console/e2e-tests/util/setup.ts
@@ -16,18 +16,20 @@
* limitations under the License.
*/
-import { createBrowserNormal } from './playwright';
+import { UNIFIED_CONSOLE_URL } from './druid';
+import { createBrowserNormal as createBrowser } from './playwright';
import { createPage } from './playwright';
-(async () => {
- const browser = await createBrowserNormal();
+export async function waitTillWebConsoleReady() {
+ const browser = await createBrowser();
try {
const page = await createPage(browser);
+ await page.goto(UNIFIED_CONSOLE_URL);
await page.waitFor('//*[contains(text(),"console will not function at the moment")]', {
visibility: 'hidden',
});
} finally {
await browser.close();
}
-})();
+}
diff --git a/web-console/jest.e2e.config.js b/web-console/jest.e2e.config.js
index a2321fe..b541118 100644
--- a/web-console/jest.e2e.config.js
+++ b/web-console/jest.e2e.config.js
@@ -21,8 +21,5 @@ const common = require('./jest.common.config');
module.exports = Object.assign(common, {
"testMatch": [
"**/?(*.)+(spec).ts?(x)"
- ],
- "setupFilesAfterEnv": [
- "<rootDir>e2e-tests/util/setup.ts"
- ],
+ ]
});
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org