You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ma...@apache.org on 2024/02/15 08:53:18 UTC
(superset) branch supersetbot-docker created (now 29cf485e7b)
This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a change to branch supersetbot-docker
in repository https://gitbox.apache.org/repos/asf/superset.git
at 29cf485e7b YUP
This branch includes the following new commits:
new 014cb9ee7d format JSON
new 29cf485e7b YUP
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(superset) 02/02: YUP
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a commit to branch supersetbot-docker
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 29cf485e7bdfd157931047cc1e0b37be48cdd26f
Author: Maxime Beauchemin <ma...@gmail.com>
AuthorDate: Wed Feb 14 16:35:59 2024 -0800
YUP
---
.github/supersetbot/src/cli.js | 18 ++
.github/supersetbot/src/docker.js | 141 ++++++++++++
.github/supersetbot/src/docker.test.js | 244 +++++++++++++++++++++
.github/supersetbot/src/index.js | 16 +-
.../src/{utils.test.js => utils.index.js} | 0
.github/supersetbot/src/utils.js | 42 +++-
6 files changed, 449 insertions(+), 12 deletions(-)
diff --git a/.github/supersetbot/src/cli.js b/.github/supersetbot/src/cli.js
index a784a98f60..3d7c6b8f66 100755
--- a/.github/supersetbot/src/cli.js
+++ b/.github/supersetbot/src/cli.js
@@ -18,6 +18,8 @@
*/
import { Command } from 'commander';
import * as commands from './commands.js';
+import * as docker from './docker.js';
+import * as utils from './utils.js';
export default function getCLI(envContext) {
const program = new Command();
@@ -58,6 +60,22 @@ export default function getCLI(envContext) {
});
await wrapped(opts.repo, opts.issue, envContext);
});
+ program.command('docker')
+ .option('-p, --preset <preset>', 'Build preset', /^(lean|dev|dockerize|websocket|py310|ci)$/i, 'lean')
+ .option('-c, --context <context>', 'Build context', /^(push|pull_request|release)$/i, 'local')
+ .option('-b, --context-ref <ref>', 'Reference to the PR, release, or branch')
+ .option('-l, --platform <platform...>', 'Platforms (multiple values allowed)', /^(linux\/arm64|linux\/amd64)$/i, ['linux/amd64'])
+ .option('-d, --dry-run', 'Run the command in dry-run mode')
+ .option('-f, --force-latest', 'Force the "latest" tag on the release')
+ .option('-v, --verbose', 'Print more info')
+ .action(function () {
+ const opts = envContext.processOptions(this, ['repo']);
+ const cmd = docker.getDockerCommand(opts);
+ console.log(cmd);
+ if (!opts.dryRun) {
+ utils.runShellCommand(cmd);
+ }
+ });
return program;
}
diff --git a/.github/supersetbot/src/docker.js b/.github/supersetbot/src/docker.js
new file mode 100644
index 0000000000..028698e0d7
--- /dev/null
+++ b/.github/supersetbot/src/docker.js
@@ -0,0 +1,141 @@
+import { spawnSync } from 'child_process';
+
+const REPO = 'apache/superset';
+const CACHE_REPO = `${REPO}-cache`;
+const BASE_PY_IMAGE = '3.9-slim-bookworm';
+
+export function runCmd(command, raiseOnFailure = true) {
+ const { stdout, stderr } = spawnSync(command, { shell: true, encoding: 'utf-8', env: process.env });
+
+ if (stderr && raiseOnFailure) {
+ throw new Error(stderr);
+ }
+ return stdout;
+}
+
+function getGitSha() {
+ return runCmd('git rev-parse HEAD').trim();
+}
+
+function getBuildContextRef(buildContext) {
+ const event = buildContext || process.env.GITHUB_EVENT_NAME;
+ const githubRef = process.env.GITHUB_REF || '';
+
+ if (event === 'pull_request') {
+ const githubHeadRef = process.env.GITHUB_HEAD_REF || '';
+ return githubHeadRef.replace(/[^a-zA-Z0-9]/g, '-').slice(0, 40);
+ } if (event === 'release') {
+ return githubRef.replace('refs/tags/', '').slice(0, 40);
+ } if (event === 'push') {
+ return githubRef.replace('refs/heads/', '').replace(/[^a-zA-Z0-9]/g, '-').slice(0, 40);
+ }
+ return '';
+}
+
+export function isLatestRelease(release) {
+ const output = runCmd(`../../scripts/tag_latest_release.sh ${release} --dry-run`, false) || '';
+ return output.includes('SKIP_TAG::false');
+}
+
+function makeDockerTag(parts) {
+ return `${REPO}:${parts.filter((part) => part).join('-')}`;
+}
+
+export function getDockerTags({
+ preset, platforms, sha, buildContext, buildContextRef, forceLatest = false,
+}) {
+ const tags = new Set();
+ const tagChunks = [];
+
+ const isLatest = isLatestRelease(buildContextRef);
+
+ if (preset !== 'lean') {
+ tagChunks.push(preset);
+ }
+
+ if (platforms.length === 1) {
+ const platform = platforms[0];
+ const shortBuildPlatform = platform.replace('linux/', '').replace('64', '');
+ if (shortBuildPlatform !== 'amd') {
+ tagChunks.push(shortBuildPlatform);
+ }
+ }
+
+ tags.add(makeDockerTag([sha, ...tagChunks]));
+ tags.add(makeDockerTag([sha.slice(0, 7), ...tagChunks]));
+
+ if (buildContext === 'release') {
+ tags.add(makeDockerTag([buildContextRef, ...tagChunks]));
+ if (isLatest || forceLatest) {
+ tags.add(makeDockerTag(['latest', ...tagChunks]));
+ }
+ } else if (buildContext === 'push' && buildContextRef === 'master') {
+ tags.add(makeDockerTag(['master', ...tagChunks]));
+ } else if (buildContext === 'pull_request') {
+ tags.add(makeDockerTag([`pr-${buildContextRef}`, ...tagChunks]));
+ }
+
+ return [...tags];
+}
+
+export function getDockerCommand({
+ preset, platform, isAuthenticated, buildContext, buildContextRef, forceLatest = false,
+}) {
+ const platforms = platform;
+
+ let buildTarget = '';
+ let pyVer = BASE_PY_IMAGE;
+ let dockerContext = '.';
+
+ if (preset === 'dev') {
+ buildTarget = 'dev';
+ } else if (preset === 'lean') {
+ buildTarget = 'lean';
+ } else if (preset === 'py310') {
+ buildTarget = 'lean';
+ pyVer = '3.10-slim-bookworm';
+ } else if (preset === 'websocket') {
+ dockerContext = 'superset-websocket';
+ } else if (preset === 'ci') {
+ buildTarget = 'ci';
+ } else if (preset === 'dockerize') {
+ dockerContext = '-f dockerize.Dockerfile .';
+ } else {
+ console.error(`Invalid build preset: ${preset}`);
+ process.exit(1);
+ }
+
+ let ref = buildContextRef;
+ if (!ref) {
+ ref = getBuildContextRef(buildContext);
+ }
+ const sha = getGitSha();
+ const tags = getDockerTags({
+ preset, platforms, sha, buildContext, buildContextRef: ref, forceLatest,
+ }).map((tag) => `-t ${tag}`).join(' \\\n ');
+ const dockerArgs = isAuthenticated ? '--push' : '--load';
+ const targetArgument = buildTarget ? `--target ${buildTarget}` : '';
+ const cacheRef = `${CACHE_REPO}:${pyVer}${platforms.length === 1 ? `-${platforms[0].replace('linux/', '').replace('64', '')}` : ''}`;
+ const platformArg = `--platform ${platforms.join(',')}`;
+ const cacheFromArg = `--cache-from=type=registry,ref=${cacheRef}`;
+ const cacheToArg = isAuthenticated ? `--cache-to=type=registry,mode=max,ref=${cacheRef}` : '';
+ const buildArg = pyVer ? `--build-arg PY_VER=${pyVer}` : '';
+ const actor = process.env.GITHUB_ACTOR;
+
+ return `
+ docker buildx build \\
+ ${dockerArgs} \\
+ ${tags} \\
+ ${cacheFromArg} \\
+ ${cacheToArg} \\
+ ${targetArgument} \\
+ ${buildArg} \\
+ ${platformArg} \\
+ --label sha=${sha} \\
+ --label target=${buildTarget} \\
+ --label build_trigger=${ref} \\
+ --label base=${pyVer} \\
+ --label build_actor=${actor} \\
+ ${dockerContext}
+ `;
+}
diff --git a/.github/supersetbot/src/docker.test.js b/.github/supersetbot/src/docker.test.js
new file mode 100644
index 0000000000..7b3c8ee3e7
--- /dev/null
+++ b/.github/supersetbot/src/docker.test.js
@@ -0,0 +1,244 @@
+import * as dockerUtils from './docker.js';
+
+const SHA = '22e7c602b9aa321ec7e0df4bb0033048664dcdf0';
+const PR_ID = '666';
+const OLD_REL = '2.1.0';
+const NEW_REL = '2.1.1';
+const REPO = 'apache/superset';
+
+beforeEach(() => {
+ process.env.TEST_ENV = 'true';
+});
+
+afterEach(() => {
+ delete process.env.TEST_ENV;
+});
+
+describe('isLatestRelease', () => {
+ test.each([
+ ['2.1.0', false],
+ ['2.1.1', true],
+ ['1.0.0', false],
+ ['3.0.0', true],
+ ])('returns %s for release %s', (release, expectedBool) => {
+ expect(dockerUtils.isLatestRelease(release)).toBe(expectedBool);
+ });
+});
+
+describe('getDockerTags', () => {
+ test.each([
+ // PRs
+ [
+ 'lean',
+ ['linux/arm64'],
+ SHA,
+ 'pull_request',
+ PR_ID,
+ [`${REPO}:22e7c60-arm`, `${REPO}:${SHA}-arm`, `${REPO}:pr-${PR_ID}-arm`],
+ ],
+ [
+ 'ci',
+ ['linux/amd64'],
+ SHA,
+ 'pull_request',
+ PR_ID,
+ [`${REPO}:22e7c60-ci`, `${REPO}:${SHA}-ci`, `${REPO}:pr-${PR_ID}-ci`],
+ ],
+ [
+ 'lean',
+ ['linux/amd64'],
+ SHA,
+ 'pull_request',
+ PR_ID,
+ [`${REPO}:22e7c60`, `${REPO}:${SHA}`, `${REPO}:pr-${PR_ID}`],
+ ],
+ [
+ 'dev',
+ ['linux/arm64'],
+ SHA,
+ 'pull_request',
+ PR_ID,
+ [
+ `${REPO}:22e7c60-dev-arm`,
+ `${REPO}:${SHA}-dev-arm`,
+ `${REPO}:pr-${PR_ID}-dev-arm`,
+ ],
+ ],
+ [
+ 'dev',
+ ['linux/amd64'],
+ SHA,
+ 'pull_request',
+ PR_ID,
+ [`${REPO}:22e7c60-dev`, `${REPO}:${SHA}-dev`, `${REPO}:pr-${PR_ID}-dev`],
+ ],
+ // old releases
+ [
+ 'lean',
+ ['linux/arm64'],
+ SHA,
+ 'release',
+ OLD_REL,
+ [`${REPO}:22e7c60-arm`, `${REPO}:${SHA}-arm`, `${REPO}:${OLD_REL}-arm`],
+ ],
+ [
+ 'lean',
+ ['linux/amd64'],
+ SHA,
+ 'release',
+ OLD_REL,
+ [`${REPO}:22e7c60`, `${REPO}:${SHA}`, `${REPO}:${OLD_REL}`],
+ ],
+ [
+ 'dev',
+ ['linux/arm64'],
+ SHA,
+ 'release',
+ OLD_REL,
+ [
+ `${REPO}:22e7c60-dev-arm`,
+ `${REPO}:${SHA}-dev-arm`,
+ `${REPO}:${OLD_REL}-dev-arm`,
+ ],
+ ],
+ [
+ 'dev',
+ ['linux/amd64'],
+ SHA,
+ 'release',
+ OLD_REL,
+ [`${REPO}:22e7c60-dev`, `${REPO}:${SHA}-dev`, `${REPO}:${OLD_REL}-dev`],
+ ],
+ // new releases
+ [
+ 'lean',
+ ['linux/arm64'],
+ SHA,
+ 'release',
+ NEW_REL,
+ [
+ `${REPO}:22e7c60-arm`,
+ `${REPO}:${SHA}-arm`,
+ `${REPO}:${NEW_REL}-arm`,
+ `${REPO}:latest-arm`,
+ ],
+ ],
+ [
+ 'lean',
+ ['linux/amd64'],
+ SHA,
+ 'release',
+ NEW_REL,
+ [`${REPO}:22e7c60`, `${REPO}:${SHA}`, `${REPO}:${NEW_REL}`, `${REPO}:latest`],
+ ],
+ [
+ 'dev',
+ ['linux/arm64'],
+ SHA,
+ 'release',
+ NEW_REL,
+ [
+ `${REPO}:22e7c60-dev-arm`,
+ `${REPO}:${SHA}-dev-arm`,
+ `${REPO}:${NEW_REL}-dev-arm`,
+ `${REPO}:latest-dev-arm`,
+ ],
+ ],
+ [
+ 'dev',
+ ['linux/amd64'],
+ SHA,
+ 'release',
+ NEW_REL,
+ [
+ `${REPO}:22e7c60-dev`,
+ `${REPO}:${SHA}-dev`,
+ `${REPO}:${NEW_REL}-dev`,
+ `${REPO}:latest-dev`,
+ ],
+ ],
+ // merge on master
+ [
+ 'lean',
+ ['linux/arm64'],
+ SHA,
+ 'push',
+ 'master',
+ [`${REPO}:22e7c60-arm`, `${REPO}:${SHA}-arm`, `${REPO}:master-arm`],
+ ],
+ [
+ 'lean',
+ ['linux/amd64'],
+ SHA,
+ 'push',
+ 'master',
+ [`${REPO}:22e7c60`, `${REPO}:${SHA}`, `${REPO}:master`],
+ ],
+ [
+ 'dev',
+ ['linux/arm64'],
+ SHA,
+ 'push',
+ 'master',
+ [
+ `${REPO}:22e7c60-dev-arm`,
+ `${REPO}:${SHA}-dev-arm`,
+ `${REPO}:master-dev-arm`,
+ ],
+ ],
+ [
+ 'dev',
+ ['linux/amd64'],
+ SHA,
+ 'push',
+ 'master',
+ [`${REPO}:22e7c60-dev`, `${REPO}:${SHA}-dev`, `${REPO}:master-dev`],
+ ],
+
+ ])('returns expected tags', (preset, platforms, sha, buildContext, buildContextRef, expectedTags) => {
+ const tags = dockerUtils.getDockerTags({
+ preset, platforms, sha, buildContext, buildContextRef,
+ });
+ expect(tags).toEqual(expect.arrayContaining(expectedTags));
+ });
+});
+
+describe('getDockerCommand', () => {
+ test.each([
+ [
+ 'lean',
+ ['linux/amd64'],
+ true,
+ SHA,
+ 'push',
+ 'master',
+ ['--push', `-t ${REPO}:master `],
+ ],
+ [
+ 'dev',
+ ['linux/amd64'],
+ false,
+ SHA,
+ 'push',
+ 'master',
+ ['--load', `-t ${REPO}:master-dev `],
+ ],
+ // multi-platform
+ [
+ 'lean',
+ ['linux/arm64', 'linux/amd64'],
+ true,
+ SHA,
+ 'push',
+ 'master',
+ ['--platform linux/arm64,linux/amd64'],
+ ],
+ ])('returns expected docker command', (preset, platform, isAuthenticated, sha, buildContext, buildContextRef, contains) => {
+ const cmd = dockerUtils.getDockerCommand({
+ preset, platform, isAuthenticated, sha, buildContext, buildContextRef,
+ });
+ contains.forEach((expectedSubstring) => {
+ expect(cmd).toContain(expectedSubstring);
+ });
+ });
+});
diff --git a/.github/supersetbot/src/index.js b/.github/supersetbot/src/index.js
index b518c35ae1..2246e4d5bc 100644
--- a/.github/supersetbot/src/index.js
+++ b/.github/supersetbot/src/index.js
@@ -16,6 +16,20 @@
* specific language governing permissions and limitations
* under the License.
*/
-import { runCommandFromGithubAction } from './utils.js';
+import { parseArgsStringToArgv } from 'string-argv';
+
+import getCLI from './cli.js';
+import Context from './context.js';
+
+async function runCommandFromGithubAction(rawCommand) {
+ const envContext = new Context('GHA');
+ const cli = getCLI(envContext);
+
+ // Make rawCommand look like argv
+ const cmd = rawCommand.trim().replace('@supersetbot', 'supersetbot');
+ const args = parseArgsStringToArgv(cmd);
+ await cli.parseAsync(['node', ...args]);
+ await envContext.onDone();
+}
export { runCommandFromGithubAction };
diff --git a/.github/supersetbot/src/utils.test.js b/.github/supersetbot/src/utils.index.js
similarity index 100%
rename from .github/supersetbot/src/utils.test.js
rename to .github/supersetbot/src/utils.index.js
diff --git a/.github/supersetbot/src/utils.js b/.github/supersetbot/src/utils.js
index e0cc6a2d62..f0d44fca52 100644
--- a/.github/supersetbot/src/utils.js
+++ b/.github/supersetbot/src/utils.js
@@ -17,17 +17,37 @@
* under the License.
*/
-import { parseArgsStringToArgv } from 'string-argv';
-import Context from './context.js';
-import getCLI from './cli.js';
+import { spawn } from 'child_process';
-export async function runCommandFromGithubAction(rawCommand) {
- const envContext = new Context('GHA');
- const cli = getCLI(envContext);
+export function runShellCommand(command) {
+ return new Promise((resolve, reject) => {
+ // Split the command string into an array of arguments
+ const args = command.split(/\s+/);
+ const childProcess = spawn(args.shift(), args);
- // Make rawCommand look like argv
- const cmd = rawCommand.trim().replace('@supersetbot', 'supersetbot');
- const args = parseArgsStringToArgv(cmd);
- await cli.parseAsync(['node', ...args]);
- await envContext.onDone();
+ let stdoutData = '';
+ let stderrData = '';
+
+ // Capture stdout data
+ childProcess.stdout.on('data', (data) => {
+ stdoutData += data;
+ console.log(`stdout: ${data}`);
+ });
+
+ // Capture stderr data
+ childProcess.stderr.on('data', (data) => {
+ stderrData += data;
+ console.error(`stderr: ${data}`);
+ });
+
+ // Handle process exit
+ childProcess.on('close', (code) => {
+ console.log(`child process exited with code ${code}`);
+ if (code === 0) {
+ resolve(stdoutData);
+ } else {
+ reject(new Error(`Command failed with code ${code}: ${stderrData}`));
+ }
+ });
+ });
}
(superset) 01/02: format JSON
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a commit to branch supersetbot-docker
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 014cb9ee7de54cc1e6cd8805aa103e2eb4640301
Author: Maxime Beauchemin <ma...@gmail.com>
AuthorDate: Wed Feb 14 15:27:00 2024 -0800
format JSON
---
.github/supersetbot/.babelrc | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.github/supersetbot/.babelrc b/.github/supersetbot/.babelrc
index 3c3d99ddbb..6c8fbcda82 100644
--- a/.github/supersetbot/.babelrc
+++ b/.github/supersetbot/.babelrc
@@ -1,5 +1,5 @@
{
- "presets": [
- ["@babel/preset-env", {"targets": {"node": "current"}}]
- ]
+ "presets": [
+ ["@babel/preset-env", {"targets": {"node": "current"}}]
+ ]
}