You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by rk...@apache.org on 2024/01/03 21:25:45 UTC
(incubator-sdap-nexus) branch SDAP-497 created (now f810653)
This is an automated email from the ASF dual-hosted git repository.
rkk pushed a change to branch SDAP-497
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
at f810653 Build script
This branch includes the following new commits:
new f810653 Build script
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(incubator-sdap-nexus) 01/01: Build script
Posted by rk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rkk pushed a commit to branch SDAP-497
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit f8106534c42b7a4b1e7978cb1bb43c04c0cbe1b7
Author: rileykk <ri...@jpl.nasa.gov>
AuthorDate: Wed Jan 3 13:25:36 2024 -0800
Build script
---
CHANGELOG.md | 1 +
build/build.py | 316 +++++++++++++++++++++++++++++++++++++++++++++++++
build/requirements.txt | 1 +
3 files changed, 318 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 01d6272..75be4cd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- SDAP-473: Added support for matchup job prioritization
- SDAP-483: Added `.asf.yaml` to configure Jira auto-linking.
- SDAP-487: Added script to migrate existing `doms.doms_data` data to new schema.
+- SDAP-497: Added convenience script to download official source distributions and build images from them.
### Changed
- SDAP-453: Updated results storage and retrieval to support output JSON from `/cdmsresults` that matches output from `/match_spark`.
- **NOTE:** Deploying these changes to an existing SDAP deployment will require modifying the Cassandra database with stored results. There is a script to do so at `/tools/update-doms-data-schema/update.py`
diff --git a/build/build.py b/build/build.py
new file mode 100644
index 0000000..f54de1c
--- /dev/null
+++ b/build/build.py
@@ -0,0 +1,316 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import shutil
+import subprocess
+import tempfile
+import textwrap
+from urllib.parse import urljoin
+
+import requests
+
+DOCKER = shutil.which('docker')
+
+NEXUS_TARBALL = 'apache-sdap-nexus-{}-incubating-src.tar.gz'
+INGESTER_TARBALL = 'apache-sdap-ingester-{}-incubating-src.tar.gz'
+
+NEXUS_DIR = 'apache-sdap-nexus-{}-incubating-src'
+INGESTER_DIR = 'apache-sdap-ingester-{}-incubating-src'
+
+if DOCKER is None:
+ raise OSError('docker command could not be found in PATH')
+
+
+def build_cmd(tag, context, dockerfile='', cache=True):
+ command = [DOCKER, 'build', context]
+
+ if dockerfile != '':
+ command.extend(['-f', os.path.join(context, dockerfile)])
+
+ command.extend(['-t', tag])
+
+ if not cache:
+ command.append('--no-cache')
+
+ return command
+
+
+def run(cmd, suppress_output=False, err_on_fail=True):
+ stdout = subprocess.DEVNULL if suppress_output else None
+
+ p = subprocess.Popen(cmd, stdout=stdout, stderr=subprocess.STDOUT)
+
+ p.wait()
+
+ if err_on_fail and p.returncode != 0:
+ raise OSError(f'Subprocess returned nonzero: {p.returncode}')
+
+
+def yes_no(prompt):
+ do_continue = input(prompt).lower()
+
+ while do_continue not in ['', 'y', 'n']:
+ do_continue = input(prompt).lower()
+
+ return do_continue in ['', 'y']
+
+
+def get_input(prompt):
+ while True:
+ response = input(prompt)
+
+ if yes_no(f'Confirm: "{response}" [Y]/N '):
+ return response
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ epilog="With the exception of the --skip-nexus, --skip-ingester, and --tag-suffix options, the user will be "
+ "prompted to set options at runtime."
+ )
+
+ parser.add_argument(
+ '--nexus-version',
+ dest='v_nexus',
+ help='Version of Nexus to download and build',
+ metavar='VERSION'
+ )
+
+ parser.add_argument(
+ '--ingester-version',
+ dest='v_ingester',
+ help='Version of Ingester to download and build',
+ metavar='VERSION'
+ )
+
+ parser.add_argument(
+ '--docker-registry',
+ dest='registry',
+ help='Docker registry to tag images with. Important if you want to push the images.'
+ )
+
+ cache = parser.add_mutually_exclusive_group(required=False)
+
+ cache.add_argument(
+ '--no-cache',
+ dest='cache',
+ action='store_false',
+ help='Don\'t use build cache'
+ )
+
+ cache.add_argument(
+ '--cache',
+ dest='cache',
+ action='store_true',
+ help='Use build cache'
+ )
+
+ push = parser.add_mutually_exclusive_group(required=False)
+
+ push.add_argument(
+ '--push',
+ dest='push',
+ action='store_true',
+ help='Push images after building'
+ )
+
+ push.add_argument(
+ '--no-push',
+ dest='push',
+ action='store_false',
+ help='Don\'t push images after building'
+ )
+
+ parser.add_argument(
+ '--dl-url',
+ dest='url',
+ help='Root url to download tarballs from. Eg: https://dist.apache.org/repos/dist/dev/incubator/sdap/apache-sdap-1.2.0-rc3/',
+ )
+
+ parser.add_argument(
+ '--skip-nexus',
+ dest='skip_nexus',
+ action='store_true',
+ help='Don\'t build Nexus webapp, Solr cloud & Solr cloud init images'
+ )
+
+ parser.add_argument(
+ '--skip-ingester',
+ dest='skip_ingester',
+ action='store_true',
+ help='Don\'t build Collection Manager & Granule Ingester images'
+ )
+
+ parser.add_argument(
+ '--tag-suffix',
+ dest='tag_suffix',
+ help='Suffix to append to image tags. For example, if you want to build images with tags ending with "-rc0" '
+ 'etc',
+ metavar='STRING'
+ )
+
+ parser.set_defaults(cache=None, push=None)
+
+ args = parser.parse_args()
+
+ v_nexus, v_ingester, registry, cache, push = args.v_nexus, args.v_ingester, args.registry, args.cache, args.push
+
+ root_url = args.url
+
+ if root_url is None:
+ root_url = get_input('Enter URL to download release from\n'
+ '(Eg: https://dist.apache.org/repos/dist/dev/incubator/sdap/apache-sdap-1.2.0-rc3/) : ')
+
+ if v_nexus is None and not args.skip_nexus:
+ v_nexus = get_input('Enter NEXUS version to build: ')
+
+ if v_ingester is None and not args.skip_ingester:
+ v_ingester = get_input('Enter Ingester version to build: ')
+
+ if registry is None:
+ registry = get_input('Enter Docker image registry: ')
+
+ if cache is None:
+ cache = yes_no('Use Docker build cache? [Y]/N: ')
+
+ if push is None:
+ push = yes_no('Push built images? [Y]/N: ')
+
+ print('Downloading release tarballs...')
+
+ extract_dir = tempfile.TemporaryDirectory()
+
+ nexus_tarball = os.path.join(extract_dir.name, NEXUS_TARBALL.format(v_nexus))
+ ingester_tarball = os.path.join(extract_dir.name, INGESTER_TARBALL.format(v_ingester))
+
+ response = requests.get(urljoin(root_url, NEXUS_TARBALL.format(v_nexus)))
+ response.raise_for_status()
+
+ with open(nexus_tarball, 'wb') as fp:
+ fp.write(response.content)
+ fp.flush()
+
+ response = requests.get(urljoin(root_url, INGESTER_TARBALL.format(v_ingester)))
+ response.raise_for_status()
+
+ with open(ingester_tarball, 'wb') as fp:
+ fp.write(response.content)
+ fp.flush()
+
+ print('Extracting release source files...')
+
+ run(
+ ['tar', 'xvf', nexus_tarball, '-C', extract_dir.name],
+ suppress_output=True
+ )
+
+ run(
+ ['tar', 'xvf', ingester_tarball, '-C', extract_dir.name],
+ suppress_output=True
+ )
+
+ shutil.move(
+ os.path.join(extract_dir.name, 'Apache-SDAP', NEXUS_DIR.format(v_nexus)),
+ os.path.join(extract_dir.name, 'nexus')
+ )
+
+ shutil.move(
+ os.path.join(extract_dir.name, 'Apache-SDAP', INGESTER_DIR.format(v_ingester)),
+ os.path.join(extract_dir.name, 'ingester')
+ )
+
+ os.environ['DOCKER_DEFAULT_PLATFORM'] = 'linux/amd64'
+
+ built_images = []
+
+ def tag(s):
+ if args.tag_suffix is not None:
+ return f'{s}-{args.tag_suffix}'
+ else:
+ return s
+
+ if not args.skip_ingester:
+ print('Building ingester images...')
+
+ cm_tag = tag(f'{registry}/sdap-collection-manager:{v_ingester}')
+
+ run(build_cmd(
+ cm_tag,
+ os.path.join(extract_dir.name, 'ingester'),
+ dockerfile='collection_manager/docker/Dockerfile',
+ cache=cache
+ ))
+
+ built_images.append(cm_tag)
+
+ gi_tag = tag(f'{registry}/sdap-granule-ingester:{v_ingester}')
+
+ run(build_cmd(
+ gi_tag,
+ os.path.join(extract_dir.name, 'ingester'),
+ dockerfile='granule_ingester/docker/Dockerfile',
+ cache=cache
+ ))
+
+ built_images.append(gi_tag)
+
+ if not args.skip_nexus:
+ solr_tag = tag(f'{registry}/sdap-solr-cloud:{v_nexus}')
+
+ run(build_cmd(
+ solr_tag,
+ os.path.join(extract_dir.name, 'nexus/docker/solr'),
+ cache=cache
+ ))
+
+ built_images.append(solr_tag)
+
+ solr_init_tag = tag(f'{registry}/sdap-solr-cloud-init:{v_nexus}')
+
+ run(build_cmd(
+ solr_init_tag,
+ os.path.join(extract_dir.name, 'nexus/docker/solr'),
+ dockerfile='cloud-init/Dockerfile',
+ cache=cache
+ ))
+
+ built_images.append(solr_init_tag)
+
+ webapp_tag = tag(f'{registry}/sdap-nexus-webapp:{v_nexus}')
+
+ run(build_cmd(
+ webapp_tag,
+ os.path.join(extract_dir.name, 'nexus'),
+ dockerfile='docker/nexus-webapp/Dockerfile',
+ cache=cache
+ ))
+
+ built_images.append(webapp_tag)
+
+ print('Image builds completed')
+
+ if push:
+ for image in built_images:
+ run(
+ [DOCKER, 'push', image]
+ )
+
+ print('done')
+
+
+if __name__ == '__main__':
+ main()
diff --git a/build/requirements.txt b/build/requirements.txt
new file mode 100644
index 0000000..f229360
--- /dev/null
+++ b/build/requirements.txt
@@ -0,0 +1 @@
+requests