You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ma...@apache.org on 2024/02/15 08:53:20 UTC
(superset) 02/02: YUP
This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a commit to branch supersetbot-docker
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 29cf485e7bdfd157931047cc1e0b37be48cdd26f
Author: Maxime Beauchemin <ma...@gmail.com>
AuthorDate: Wed Feb 14 16:35:59 2024 -0800
YUP
---
.github/supersetbot/src/cli.js | 18 ++
.github/supersetbot/src/docker.js | 141 ++++++++++++
.github/supersetbot/src/docker.test.js | 244 +++++++++++++++++++++
.github/supersetbot/src/index.js | 16 +-
.../src/{utils.test.js => utils.index.js} | 0
.github/supersetbot/src/utils.js | 42 +++-
6 files changed, 449 insertions(+), 12 deletions(-)
diff --git a/.github/supersetbot/src/cli.js b/.github/supersetbot/src/cli.js
index a784a98f60..3d7c6b8f66 100755
--- a/.github/supersetbot/src/cli.js
+++ b/.github/supersetbot/src/cli.js
@@ -18,6 +18,8 @@
*/
import { Command } from 'commander';
import * as commands from './commands.js';
+import * as docker from './docker.js';
+import * as utils from './utils.js';
export default function getCLI(envContext) {
const program = new Command();
@@ -58,6 +60,22 @@ export default function getCLI(envContext) {
});
await wrapped(opts.repo, opts.issue, envContext);
});
+ program.command('docker')
+ .option('-p, --preset <preset>', 'Build preset', /^(lean|dev|dockerize|websocket|py310|ci)$/i, 'lean')
+ .option('-c, --context <context>', 'Build context', /^(push|pull_request|release)$/i, 'local')
+ .option('-b, --context-ref <ref>', 'Reference to the PR, release, or branch')
+ .option('-l, --platform <platform...>', 'Platforms (multiple values allowed)', /^(linux\/arm64|linux\/amd64)$/i, ['linux/amd64'])
+ .option('-d, --dry-run', 'Run the command in dry-run mode')
+ .option('-f, --force-latest', 'Force the "latest" tag on the release')
+ .option('-v, --verbose', 'Print more info')
+ .action(function () {
+ const opts = envContext.processOptions(this, ['repo']);
+ const cmd = docker.getDockerCommand(opts);
+ console.log(cmd);
+ if (!opts.dryRun) {
+ utils.runShellCommand(cmd);
+ }
+ });
return program;
}
diff --git a/.github/supersetbot/src/docker.js b/.github/supersetbot/src/docker.js
new file mode 100644
index 0000000000..028698e0d7
--- /dev/null
+++ b/.github/supersetbot/src/docker.js
@@ -0,0 +1,141 @@
+import { spawnSync } from 'child_process';
+
+const REPO = 'apache/superset';
+const CACHE_REPO = `${REPO}-cache`;
+const BASE_PY_IMAGE = '3.9-slim-bookworm';
+
+export function runCmd(command, raiseOnFailure = true) {
+ const { stdout, stderr } = spawnSync(command, { shell: true, encoding: 'utf-8', env: process.env });
+
+ if (stderr && raiseOnFailure) {
+ throw new Error(stderr);
+ }
+ return stdout;
+}
+
+function getGitSha() {
+ return runCmd('git rev-parse HEAD').trim();
+}
+
+function getBuildContextRef(buildContext) {
+ const event = buildContext || process.env.GITHUB_EVENT_NAME;
+ const githubRef = process.env.GITHUB_REF || '';
+
+ if (event === 'pull_request') {
+ const githubHeadRef = process.env.GITHUB_HEAD_REF || '';
+ return githubHeadRef.replace(/[^a-zA-Z0-9]/g, '-').slice(0, 40);
+ } if (event === 'release') {
+ return githubRef.replace('refs/tags/', '').slice(0, 40);
+ } if (event === 'push') {
+ return githubRef.replace('refs/heads/', '').replace(/[^a-zA-Z0-9]/g, '-').slice(0, 40);
+ }
+ return '';
+}
+
+export function isLatestRelease(release) {
+ const output = runCmd(`../../scripts/tag_latest_release.sh ${release} --dry-run`, false) || '';
+ return output.includes('SKIP_TAG::false');
+}
+
+function makeDockerTag(parts) {
+ return `${REPO}:${parts.filter((part) => part).join('-')}`;
+}
+
+export function getDockerTags({
+ preset, platforms, sha, buildContext, buildContextRef, forceLatest = false,
+}) {
+ const tags = new Set();
+ const tagChunks = [];
+
+ const isLatest = isLatestRelease(buildContextRef);
+
+ if (preset !== 'lean') {
+ tagChunks.push(preset);
+ }
+
+ if (platforms.length === 1) {
+ const platform = platforms[0];
+ const shortBuildPlatform = platform.replace('linux/', '').replace('64', '');
+ if (shortBuildPlatform !== 'amd') {
+ tagChunks.push(shortBuildPlatform);
+ }
+ }
+
+ tags.add(makeDockerTag([sha, ...tagChunks]));
+ tags.add(makeDockerTag([sha.slice(0, 7), ...tagChunks]));
+
+ if (buildContext === 'release') {
+ tags.add(makeDockerTag([buildContextRef, ...tagChunks]));
+ if (isLatest || forceLatest) {
+ tags.add(makeDockerTag(['latest', ...tagChunks]));
+ }
+ } else if (buildContext === 'push' && buildContextRef === 'master') {
+ tags.add(makeDockerTag(['master', ...tagChunks]));
+ } else if (buildContext === 'pull_request') {
+ tags.add(makeDockerTag([`pr-${buildContextRef}`, ...tagChunks]));
+ }
+
+ return [...tags];
+}
+
+export function getDockerCommand({
+ preset, platform, isAuthenticated, buildContext, buildContextRef, forceLatest = false,
+}) {
+ const platforms = platform;
+
+ let buildTarget = '';
+ let pyVer = BASE_PY_IMAGE;
+ let dockerContext = '.';
+
+ if (preset === 'dev') {
+ buildTarget = 'dev';
+ } else if (preset === 'lean') {
+ buildTarget = 'lean';
+ } else if (preset === 'py310') {
+ buildTarget = 'lean';
+ pyVer = '3.10-slim-bookworm';
+ } else if (preset === 'websocket') {
+ dockerContext = 'superset-websocket';
+ } else if (preset === 'ci') {
+ buildTarget = 'ci';
+ } else if (preset === 'dockerize') {
+ dockerContext = '-f dockerize.Dockerfile .';
+ } else {
+ console.error(`Invalid build preset: ${preset}`);
+ process.exit(1);
+ }
+
+ let ref = buildContextRef;
+ if (!ref) {
+ ref = getBuildContextRef(buildContext);
+ }
+ const sha = getGitSha();
+ const tags = getDockerTags({
+ preset, platforms, sha, buildContext, buildContextRef: ref, forceLatest,
+ }).map((tag) => `-t ${tag}`).join(' \\\n ');
+ const dockerArgs = isAuthenticated ? '--push' : '--load';
+ const targetArgument = buildTarget ? `--target ${buildTarget}` : '';
+ const cacheRef = `${CACHE_REPO}:${pyVer}${platforms.length === 1 ? `-${platforms[0].replace('linux/', '').replace('64', '')}` : ''}`;
+ const platformArg = `--platform ${platforms.join(',')}`;
+ const cacheFromArg = `--cache-from=type=registry,ref=${cacheRef}`;
+ const cacheToArg = isAuthenticated ? `--cache-to=type=registry,mode=max,ref=${cacheRef}` : '';
+ const buildArg = pyVer ? `--build-arg PY_VER=${pyVer}` : '';
+ const actor = process.env.GITHUB_ACTOR;
+
+ return `
+ docker buildx build \\
+ ${dockerArgs} \\
+ ${tags} \\
+ ${cacheFromArg} \\
+ ${cacheToArg} \\
+ ${targetArgument} \\
+ ${buildArg} \\
+ ${platformArg} \\
+ --label sha=${sha} \\
+ --label target=${buildTarget} \\
+ --label build_trigger=${ref} \\
+ --label base=${pyVer} \\
+ --label build_actor=${actor} \\
+ ${dockerContext}
+ `;
+}
diff --git a/.github/supersetbot/src/docker.test.js b/.github/supersetbot/src/docker.test.js
new file mode 100644
index 0000000000..7b3c8ee3e7
--- /dev/null
+++ b/.github/supersetbot/src/docker.test.js
@@ -0,0 +1,244 @@
+import * as dockerUtils from './docker.js';
+
+const SHA = '22e7c602b9aa321ec7e0df4bb0033048664dcdf0';
+const PR_ID = '666';
+const OLD_REL = '2.1.0';
+const NEW_REL = '2.1.1';
+const REPO = 'apache/superset';
+
+beforeEach(() => {
+ process.env.TEST_ENV = 'true';
+});
+
+afterEach(() => {
+ delete process.env.TEST_ENV;
+});
+
+describe('isLatestRelease', () => {
+ test.each([
+ ['2.1.0', false],
+ ['2.1.1', true],
+ ['1.0.0', false],
+ ['3.0.0', true],
+ ])('returns %s for release %s', (release, expectedBool) => {
+ expect(dockerUtils.isLatestRelease(release)).toBe(expectedBool);
+ });
+});
+
+describe('getDockerTags', () => {
+ test.each([
+ // PRs
+ [
+ 'lean',
+ ['linux/arm64'],
+ SHA,
+ 'pull_request',
+ PR_ID,
+ [`${REPO}:22e7c60-arm`, `${REPO}:${SHA}-arm`, `${REPO}:pr-${PR_ID}-arm`],
+ ],
+ [
+ 'ci',
+ ['linux/amd64'],
+ SHA,
+ 'pull_request',
+ PR_ID,
+ [`${REPO}:22e7c60-ci`, `${REPO}:${SHA}-ci`, `${REPO}:pr-${PR_ID}-ci`],
+ ],
+ [
+ 'lean',
+ ['linux/amd64'],
+ SHA,
+ 'pull_request',
+ PR_ID,
+ [`${REPO}:22e7c60`, `${REPO}:${SHA}`, `${REPO}:pr-${PR_ID}`],
+ ],
+ [
+ 'dev',
+ ['linux/arm64'],
+ SHA,
+ 'pull_request',
+ PR_ID,
+ [
+ `${REPO}:22e7c60-dev-arm`,
+ `${REPO}:${SHA}-dev-arm`,
+ `${REPO}:pr-${PR_ID}-dev-arm`,
+ ],
+ ],
+ [
+ 'dev',
+ ['linux/amd64'],
+ SHA,
+ 'pull_request',
+ PR_ID,
+ [`${REPO}:22e7c60-dev`, `${REPO}:${SHA}-dev`, `${REPO}:pr-${PR_ID}-dev`],
+ ],
+ // old releases
+ [
+ 'lean',
+ ['linux/arm64'],
+ SHA,
+ 'release',
+ OLD_REL,
+ [`${REPO}:22e7c60-arm`, `${REPO}:${SHA}-arm`, `${REPO}:${OLD_REL}-arm`],
+ ],
+ [
+ 'lean',
+ ['linux/amd64'],
+ SHA,
+ 'release',
+ OLD_REL,
+ [`${REPO}:22e7c60`, `${REPO}:${SHA}`, `${REPO}:${OLD_REL}`],
+ ],
+ [
+ 'dev',
+ ['linux/arm64'],
+ SHA,
+ 'release',
+ OLD_REL,
+ [
+ `${REPO}:22e7c60-dev-arm`,
+ `${REPO}:${SHA}-dev-arm`,
+ `${REPO}:${OLD_REL}-dev-arm`,
+ ],
+ ],
+ [
+ 'dev',
+ ['linux/amd64'],
+ SHA,
+ 'release',
+ OLD_REL,
+ [`${REPO}:22e7c60-dev`, `${REPO}:${SHA}-dev`, `${REPO}:${OLD_REL}-dev`],
+ ],
+ // new releases
+ [
+ 'lean',
+ ['linux/arm64'],
+ SHA,
+ 'release',
+ NEW_REL,
+ [
+ `${REPO}:22e7c60-arm`,
+ `${REPO}:${SHA}-arm`,
+ `${REPO}:${NEW_REL}-arm`,
+ `${REPO}:latest-arm`,
+ ],
+ ],
+ [
+ 'lean',
+ ['linux/amd64'],
+ SHA,
+ 'release',
+ NEW_REL,
+ [`${REPO}:22e7c60`, `${REPO}:${SHA}`, `${REPO}:${NEW_REL}`, `${REPO}:latest`],
+ ],
+ [
+ 'dev',
+ ['linux/arm64'],
+ SHA,
+ 'release',
+ NEW_REL,
+ [
+ `${REPO}:22e7c60-dev-arm`,
+ `${REPO}:${SHA}-dev-arm`,
+ `${REPO}:${NEW_REL}-dev-arm`,
+ `${REPO}:latest-dev-arm`,
+ ],
+ ],
+ [
+ 'dev',
+ ['linux/amd64'],
+ SHA,
+ 'release',
+ NEW_REL,
+ [
+ `${REPO}:22e7c60-dev`,
+ `${REPO}:${SHA}-dev`,
+ `${REPO}:${NEW_REL}-dev`,
+ `${REPO}:latest-dev`,
+ ],
+ ],
+ // merge on master
+ [
+ 'lean',
+ ['linux/arm64'],
+ SHA,
+ 'push',
+ 'master',
+ [`${REPO}:22e7c60-arm`, `${REPO}:${SHA}-arm`, `${REPO}:master-arm`],
+ ],
+ [
+ 'lean',
+ ['linux/amd64'],
+ SHA,
+ 'push',
+ 'master',
+ [`${REPO}:22e7c60`, `${REPO}:${SHA}`, `${REPO}:master`],
+ ],
+ [
+ 'dev',
+ ['linux/arm64'],
+ SHA,
+ 'push',
+ 'master',
+ [
+ `${REPO}:22e7c60-dev-arm`,
+ `${REPO}:${SHA}-dev-arm`,
+ `${REPO}:master-dev-arm`,
+ ],
+ ],
+ [
+ 'dev',
+ ['linux/amd64'],
+ SHA,
+ 'push',
+ 'master',
+ [`${REPO}:22e7c60-dev`, `${REPO}:${SHA}-dev`, `${REPO}:master-dev`],
+ ],
+
+ ])('returns expected tags', (preset, platforms, sha, buildContext, buildContextRef, expectedTags) => {
+ const tags = dockerUtils.getDockerTags({
+ preset, platforms, sha, buildContext, buildContextRef,
+ });
+ expect(tags).toEqual(expect.arrayContaining(expectedTags));
+ });
+});
+
+describe('getDockerCommand', () => {
+ test.each([
+ [
+ 'lean',
+ ['linux/amd64'],
+ true,
+ SHA,
+ 'push',
+ 'master',
+ ['--push', `-t ${REPO}:master `],
+ ],
+ [
+ 'dev',
+ ['linux/amd64'],
+ false,
+ SHA,
+ 'push',
+ 'master',
+ ['--load', `-t ${REPO}:master-dev `],
+ ],
+ // multi-platform
+ [
+ 'lean',
+ ['linux/arm64', 'linux/amd64'],
+ true,
+ SHA,
+ 'push',
+ 'master',
+ ['--platform linux/arm64,linux/amd64'],
+ ],
+ ])('returns expected docker command', (preset, platform, isAuthenticated, sha, buildContext, buildContextRef, contains) => {
+ const cmd = dockerUtils.getDockerCommand({
+ preset, platform, isAuthenticated, sha, buildContext, buildContextRef,
+ });
+ contains.forEach((expectedSubstring) => {
+ expect(cmd).toContain(expectedSubstring);
+ });
+ });
+});
diff --git a/.github/supersetbot/src/index.js b/.github/supersetbot/src/index.js
index b518c35ae1..2246e4d5bc 100644
--- a/.github/supersetbot/src/index.js
+++ b/.github/supersetbot/src/index.js
@@ -16,6 +16,20 @@
* specific language governing permissions and limitations
* under the License.
*/
-import { runCommandFromGithubAction } from './utils.js';
+import { parseArgsStringToArgv } from 'string-argv';
+
+import getCLI from './cli.js';
+import Context from './context.js';
+
+async function runCommandFromGithubAction(rawCommand) {
+ const envContext = new Context('GHA');
+ const cli = getCLI(envContext);
+
+ // Make rawCommand look like argv
+ const cmd = rawCommand.trim().replace('@supersetbot', 'supersetbot');
+ const args = parseArgsStringToArgv(cmd);
+ await cli.parseAsync(['node', ...args]);
+ await envContext.onDone();
+}
export { runCommandFromGithubAction };
diff --git a/.github/supersetbot/src/utils.test.js b/.github/supersetbot/src/utils.index.js
similarity index 100%
rename from .github/supersetbot/src/utils.test.js
rename to .github/supersetbot/src/utils.index.js
diff --git a/.github/supersetbot/src/utils.js b/.github/supersetbot/src/utils.js
index e0cc6a2d62..f0d44fca52 100644
--- a/.github/supersetbot/src/utils.js
+++ b/.github/supersetbot/src/utils.js
@@ -17,17 +17,37 @@
* under the License.
*/
-import { parseArgsStringToArgv } from 'string-argv';
-import Context from './context.js';
-import getCLI from './cli.js';
+import { spawn } from 'child_process';
-export async function runCommandFromGithubAction(rawCommand) {
- const envContext = new Context('GHA');
- const cli = getCLI(envContext);
+export function runShellCommand(command) {
+ return new Promise((resolve, reject) => {
+ // Split the command string into an array of arguments
+ const args = command.split(/\s+/);
+ const childProcess = spawn(args.shift(), args);
- // Make rawCommand look like argv
- const cmd = rawCommand.trim().replace('@supersetbot', 'supersetbot');
- const args = parseArgsStringToArgv(cmd);
- await cli.parseAsync(['node', ...args]);
- await envContext.onDone();
+ let stdoutData = '';
+ let stderrData = '';
+
+ // Capture stdout data
+ childProcess.stdout.on('data', (data) => {
+ stdoutData += data;
+ console.log(`stdout: ${data}`);
+ });
+
+ // Capture stderr data
+ childProcess.stderr.on('data', (data) => {
+ stderrData += data;
+ console.error(`stderr: ${data}`);
+ });
+
+ // Handle process exit
+ childProcess.on('close', (code) => {
+ console.log(`child process exited with code ${code}`);
+ if (code === 0) {
+ resolve(stdoutData);
+ } else {
+ reject(new Error(`Command failed with code ${code}: ${stderrData}`));
+ }
+ });
+ });
}