You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by rk...@apache.org on 2024/01/03 21:25:46 UTC
(incubator-sdap-nexus) 01/01: Build script
This is an automated email from the ASF dual-hosted git repository.
rkk pushed a commit to branch SDAP-497
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit f8106534c42b7a4b1e7978cb1bb43c04c0cbe1b7
Author: rileykk <ri...@jpl.nasa.gov>
AuthorDate: Wed Jan 3 13:25:36 2024 -0800
Build script
---
CHANGELOG.md | 1 +
build/build.py | 316 +++++++++++++++++++++++++++++++++++++++++++++++++
build/requirements.txt | 1 +
3 files changed, 318 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 01d6272..75be4cd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- SDAP-473: Added support for matchup job prioritization
- SDAP-483: Added `.asf.yaml` to configure Jira auto-linking.
- SDAP-487: Added script to migrate existing `doms.doms_data` data to new schema.
+- SDAP-497: Added convenience script to download official source distributions and build images from them.
### Changed
- SDAP-453: Updated results storage and retrieval to support output JSON from `/cdmsresults` that matches output from `/match_spark`.
- **NOTE:** Deploying these changes to an existing SDAP deployment will require modifying the Cassandra database with stored results. There is a script to do so at `/tools/update-doms-data-schema/update.py`
diff --git a/build/build.py b/build/build.py
new file mode 100644
index 0000000..f54de1c
--- /dev/null
+++ b/build/build.py
@@ -0,0 +1,316 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import shutil
+import subprocess
+import tempfile
+import textwrap
+from urllib.parse import urljoin
+
+import requests
+
+DOCKER = shutil.which('docker')
+
+NEXUS_TARBALL = 'apache-sdap-nexus-{}-incubating-src.tar.gz'
+INGESTER_TARBALL = 'apache-sdap-ingester-{}-incubating-src.tar.gz'
+
+NEXUS_DIR = 'apache-sdap-nexus-{}-incubating-src'
+INGESTER_DIR = 'apache-sdap-ingester-{}-incubating-src'
+
+if DOCKER is None:
+ raise OSError('docker command could not be found in PATH')
+
+
+def build_cmd(tag, context, dockerfile='', cache=True):
+ command = [DOCKER, 'build', context]
+
+ if dockerfile != '':
+ command.extend(['-f', os.path.join(context, dockerfile)])
+
+ command.extend(['-t', tag])
+
+ if not cache:
+ command.append('--no-cache')
+
+ return command
+
+
+def run(cmd, suppress_output=False, err_on_fail=True):
+ stdout = subprocess.DEVNULL if suppress_output else None
+
+ p = subprocess.Popen(cmd, stdout=stdout, stderr=subprocess.STDOUT)
+
+ p.wait()
+
+ if err_on_fail and p.returncode != 0:
+ raise OSError(f'Subprocess returned nonzero: {p.returncode}')
+
+
+def yes_no(prompt):
+ do_continue = input(prompt).lower()
+
+ while do_continue not in ['', 'y', 'n']:
+ do_continue = input(prompt).lower()
+
+ return do_continue in ['', 'y']
+
+
+def get_input(prompt):
+ while True:
+ response = input(prompt)
+
+ if yes_no(f'Confirm: "{response}" [Y]/N '):
+ return response
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ epilog="With the exception of the --skip-nexus, --skip-ingester, and --tag-suffix options, the user will be "
+ "prompted to set options at runtime."
+ )
+
+ parser.add_argument(
+ '--nexus-version',
+ dest='v_nexus',
+ help='Version of Nexus to download and build',
+ metavar='VERSION'
+ )
+
+ parser.add_argument(
+ '--ingester-version',
+ dest='v_ingester',
+ help='Version of Ingester to download and build',
+ metavar='VERSION'
+ )
+
+ parser.add_argument(
+ '--docker-registry',
+ dest='registry',
+ help='Docker registry to tag images with. Important if you want to push the images.'
+ )
+
+ cache = parser.add_mutually_exclusive_group(required=False)
+
+ cache.add_argument(
+ '--no-cache',
+ dest='cache',
+ action='store_false',
+ help='Don\'t use build cache'
+ )
+
+ cache.add_argument(
+ '--cache',
+ dest='cache',
+ action='store_true',
+ help='Use build cache'
+ )
+
+ push = parser.add_mutually_exclusive_group(required=False)
+
+ push.add_argument(
+ '--push',
+ dest='push',
+ action='store_true',
+ help='Push images after building'
+ )
+
+ push.add_argument(
+ '--no-push',
+ dest='push',
+ action='store_false',
+ help='Don\'t push images after building'
+ )
+
+ parser.add_argument(
+ '--dl-url',
+ dest='url',
+ help='Root url to download tarballs from. Eg: https://dist.apache.org/repos/dist/dev/incubator/sdap/apache-sdap-1.2.0-rc3/',
+ )
+
+ parser.add_argument(
+ '--skip-nexus',
+ dest='skip_nexus',
+ action='store_true',
+ help='Don\'t build Nexus webapp, Solr cloud & Solr cloud init images'
+ )
+
+ parser.add_argument(
+ '--skip-ingester',
+ dest='skip_ingester',
+ action='store_true',
+ help='Don\'t build Collection Manager & Granule Ingester images'
+ )
+
+ parser.add_argument(
+ '--tag-suffix',
+ dest='tag_suffix',
+ help='Suffix to append to image tags. For example, if you want to build images with tags ending with "-rc0" '
+ 'etc',
+ metavar='STRING'
+ )
+
+ parser.set_defaults(cache=None, push=None)
+
+ args = parser.parse_args()
+
+ v_nexus, v_ingester, registry, cache, push = args.v_nexus, args.v_ingester, args.registry, args.cache, args.push
+
+ root_url = args.url
+
+ if root_url is None:
+ root_url = get_input('Enter URL to download release from\n'
+ '(Eg: https://dist.apache.org/repos/dist/dev/incubator/sdap/apache-sdap-1.2.0-rc3/) : ')
+
+ if v_nexus is None and not args.skip_nexus:
+ v_nexus = get_input('Enter NEXUS version to build: ')
+
+ if v_ingester is None and not args.skip_ingester:
+ v_ingester = get_input('Enter Ingester version to build: ')
+
+ if registry is None:
+ registry = get_input('Enter Docker image registry: ')
+
+ if cache is None:
+ cache = yes_no('Use Docker build cache? [Y]/N: ')
+
+ if push is None:
+ push = yes_no('Push built images? [Y]/N: ')
+
+ print('Downloading release tarballs...')
+
+ extract_dir = tempfile.TemporaryDirectory()
+
+ nexus_tarball = os.path.join(extract_dir.name, NEXUS_TARBALL.format(v_nexus))
+ ingester_tarball = os.path.join(extract_dir.name, INGESTER_TARBALL.format(v_ingester))
+
+ response = requests.get(urljoin(root_url, NEXUS_TARBALL.format(v_nexus)))
+ response.raise_for_status()
+
+ with open(nexus_tarball, 'wb') as fp:
+ fp.write(response.content)
+ fp.flush()
+
+ response = requests.get(urljoin(root_url, INGESTER_TARBALL.format(v_ingester)))
+ response.raise_for_status()
+
+ with open(ingester_tarball, 'wb') as fp:
+ fp.write(response.content)
+ fp.flush()
+
+ print('Extracting release source files...')
+
+ run(
+ ['tar', 'xvf', nexus_tarball, '-C', extract_dir.name],
+ suppress_output=True
+ )
+
+ run(
+ ['tar', 'xvf', ingester_tarball, '-C', extract_dir.name],
+ suppress_output=True
+ )
+
+ shutil.move(
+ os.path.join(extract_dir.name, 'Apache-SDAP', NEXUS_DIR.format(v_nexus)),
+ os.path.join(extract_dir.name, 'nexus')
+ )
+
+ shutil.move(
+ os.path.join(extract_dir.name, 'Apache-SDAP', INGESTER_DIR.format(v_ingester)),
+ os.path.join(extract_dir.name, 'ingester')
+ )
+
+ os.environ['DOCKER_DEFAULT_PLATFORM'] = 'linux/amd64'
+
+ built_images = []
+
+ def tag(s):
+ if args.tag_suffix is not None:
+ return f'{s}-{args.tag_suffix}'
+ else:
+ return s
+
+ if not args.skip_ingester:
+ print('Building ingester images...')
+
+ cm_tag = tag(f'{registry}/sdap-collection-manager:{v_ingester}')
+
+ run(build_cmd(
+ cm_tag,
+ os.path.join(extract_dir.name, 'ingester'),
+ dockerfile='collection_manager/docker/Dockerfile',
+ cache=cache
+ ))
+
+ built_images.append(cm_tag)
+
+ gi_tag = tag(f'{registry}/sdap-granule-ingester:{v_ingester}')
+
+ run(build_cmd(
+ gi_tag,
+ os.path.join(extract_dir.name, 'ingester'),
+ dockerfile='granule_ingester/docker/Dockerfile',
+ cache=cache
+ ))
+
+ built_images.append(gi_tag)
+
+ if not args.skip_nexus:
+ solr_tag = tag(f'{registry}/sdap-solr-cloud:{v_nexus}')
+
+ run(build_cmd(
+ solr_tag,
+ os.path.join(extract_dir.name, 'nexus/docker/solr'),
+ cache=cache
+ ))
+
+ built_images.append(solr_tag)
+
+ solr_init_tag = tag(f'{registry}/sdap-solr-cloud-init:{v_nexus}')
+
+ run(build_cmd(
+ solr_init_tag,
+ os.path.join(extract_dir.name, 'nexus/docker/solr'),
+ dockerfile='cloud-init/Dockerfile',
+ cache=cache
+ ))
+
+ built_images.append(solr_init_tag)
+
+ webapp_tag = tag(f'{registry}/sdap-nexus-webapp:{v_nexus}')
+
+ run(build_cmd(
+ webapp_tag,
+ os.path.join(extract_dir.name, 'nexus'),
+ dockerfile='docker/nexus-webapp/Dockerfile',
+ cache=cache
+ ))
+
+ built_images.append(webapp_tag)
+
+ print('Image builds completed')
+
+ if push:
+ for image in built_images:
+ run(
+ [DOCKER, 'push', image]
+ )
+
+ print('done')
+
+
+if __name__ == '__main__':
+ main()
diff --git a/build/requirements.txt b/build/requirements.txt
new file mode 100644
index 0000000..f229360
--- /dev/null
+++ b/build/requirements.txt
@@ -0,0 +1 @@
+requests