[lucene-solr] 07/12: SOLR-14949: Ability to customize Solr Docker build (#2020)

[ma...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git

commit a649da35a571d4802f8b4cd41684abcb093669ff
Author: Houston Putman <ho...@apache.org>
AuthorDate: Tue Nov 10 10:42:38 2020 -0500

    SOLR-14949: Ability to customize Solr Docker build (#2020)
    
    Also added a gradlew helpDocker page.
---
 .github/workflows/docker-test.yml       | 44 +++++++++++++++++++++++++++
 gradle/help.gradle                      |  1 +
 help/docker.txt                         | 53 +++++++++++++++++++++++++++++++++
 solr/CHANGES.txt                        |  7 +++++
 solr/docker/build.gradle                | 24 +++++++++++++--
 solr/docker/tests/cases/version/test.sh | 45 ----------------------------
 6 files changed, 126 insertions(+), 48 deletions(-)

diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml
new file mode 100644
index 0000000..76fa714
--- /dev/null
+++ b/.github/workflows/docker-test.yml
@@ -0,0 +1,44 @@
+name: Docker Build & Test
+
+on:
+  pull_request:
+    branches:
+      - 'master'
+    paths:
+      - '.github/workflows/docker-test.yml'
+      - 'solr/bin/**'
+      - 'solr/contrib/prometheus-exporter/bin/**'
+      - 'solr/docker/**'
+      - 'solr/packaging/**'
+
+jobs:
+  test:
+    name: Build and test Docker image
+
+    runs-on: ubuntu-latest
+
+    env:
+      SOLR_DOCKER_IMAGE_REPO: github-pr/solr
+      SOLR_DOCKER_IMAGE_TAG: ${{github.event.number}}
+
+    steps:
+    # Setup
+    - uses: actions/checkout@v2
+    - name: Set up JDK 11
+      uses: actions/setup-java@v1
+      with:
+        java-version: 11
+    - name: Grant execute permission for gradlew
+      run: chmod +x gradlew
+    - uses: actions/cache@v2
+      with:
+        path: |
+          ~/.gradle/caches
+        key: ${{ runner.os }}-gradle-docker-${{ hashFiles('versions.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-gradle-docker-
+          ${{ runner.os }}-gradle-
+    - name: Build Docker image with Gradle
+      run: ./gradlew solr:docker:docker
+    - name: Run tests on Docker image
+      run: ./gradlew solr:docker:testDocker
diff --git a/gradle/help.gradle b/gradle/help.gradle
index e03d724..4c1bf7e 100644
--- a/gradle/help.gradle
+++ b/gradle/help.gradle
@@ -29,6 +29,7 @@ configure(rootProject) {
       ["Git", "help/git.txt", "Git assistance and guides."],
       ["ValidateLogCalls", "help/validateLogCalls.txt", "How to use logging calls efficiently."],
       ["IDEs", "help/IDEs.txt", "IDE support."],
+      ["Docker", "help/docker.txt", "Building Solr Docker images."],
   ]
 
   helpFiles.each { section, path, sectionInfo ->
diff --git a/help/docker.txt b/help/docker.txt
new file mode 100644
index 0000000..8438713
--- /dev/null
+++ b/help/docker.txt
@@ -0,0 +1,53 @@
+Docker Images for Solr
+======================
+
+Solr docker images are built using Palantir's Docker Gradle plugin, https://github.com/palantir/gradle-docker.
+
+Common Inputs
+-------------
+
+The docker image and its tag can be customized via the following options, all accepted via both Environment Variables and Gradle Properties.
+
+Docker Image Repository:
+   Default: "apache/solr"
+   EnvVar: SOLR_DOCKER_IMAGE_REPO
+   Gradle Property: -Psolr.docker.imageRepo
+
+Docker Image Tag:
+   Default: the Solr version, e.g. "9.0.0-SNAPSHOT"
+   EnvVar: SOLR_DOCKER_IMAGE_TAG
+   Gradle Property: -Psolr.docker.imageTag
+
+Docker Image Name: (Use this to explicitly set a whole image name. If given, the image repo and image version options above are ignored.)
+   Default: {image_repo}/{image_tag} (both options provided above, with defaults)
+   EnvVar: SOLR_DOCKER_IMAGE_NAME
+   Gradle Property: -Psolr.docker.imageName
+
+Building
+--------
+
+In order to build the Solr Docker image, run:
+
+gradlew docker
+
+The docker build task accepts the following inputs, in addition to the common inputs listed above:
+
+Base Docker Image: (The docker image used for the "FROM" in the Solr Dockerfile)
+   Default: "openjdk:11-jre-slim"
+   EnvVar: SOLR_DOCKER_BASE_IMAGE
+   Gradle Property: -Psolr.docker.baseImage
+
+Testing
+-------
+
+To test the docker image, run:
+
+gradlew dockerTest
+
+If a custom docker image name was used, via one of the common inputs described above, then the same input must be used while testing.
+
+You can also specify an explicit list of tests to run, or an explicit list of tests to ignore.
+Both inputs are optional, and by default all tests will be run.
+
+gradlew testDocker --tests create_core,demo
+gradlew testDocker --ignore demo-tini,initdb
\ No newline at end of file
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index ad0c1f6..f09e496 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -136,6 +136,13 @@ Other Changes
 
 * SOLR-14957: Add Prometheus Exporter to docker PATH. Fix classpath issues. (Houston Putman)
 
+* SOLR-14912: Clean up solr-extraction contrib to produce solr-extraction-* jar
+  (instead of solr-cell-*). (Dawid Weiss)
+
+* SOLR-14978: Enable OOM Killer Script in Solr Foreground. Simplify getting heap dumps on OOM. (Mike Drob, Houston Putman)
+
+* SOLR-14949: Ability to customize docker image name/base image (Houston Putman)
+
 Bug Fixes
 ---------------------
 * SOLR-14546: Fix for a relatively hard to hit issue in OverseerTaskProcessor that could lead to out of order execution
diff --git a/solr/docker/build.gradle b/solr/docker/build.gradle
index a4cdeea..0aaefff 100644
--- a/solr/docker/build.gradle
+++ b/solr/docker/build.gradle
@@ -15,6 +15,9 @@
  * limitations under the License.
  */
 
+import com.google.common.base.Preconditions
+import com.google.common.base.Strings
+
 apply plugin: 'base'
 apply plugin: 'com.palantir.docker'
 
@@ -31,8 +34,10 @@ dependencies {
   docker dockerPackage
 }
 
-def dockerImageName = "apache/solr:${version}"
-def baseDockerImage = 'openjdk:11-jre-slim'
+def dockerImageRepo = propertyOrEnvOrDefault("solr.docker.imageRepo", "SOLR_DOCKER_IMAGE_REPO", "apache/solr")
+def dockerImageTag = propertyOrEnvOrDefault("solr.docker.imageTag", "SOLR_DOCKER_IMAGE_TAG", "${version}")
+def dockerImageName = propertyOrEnvOrDefault("solr.docker.imageName", "SOLR_DOCKER_IMAGE_NAME", "${dockerImageRepo}:${dockerImageTag}")
+def baseDockerImage = propertyOrEnvOrDefault("solr.docker.baseImage", "SOLR_DOCKER_BASE_IMAGE", 'openjdk:11-jre-slim')
 
 docker {
   name = dockerImageName
@@ -53,12 +58,22 @@ tasks.docker {
 }
 
 abstract class DockerTestSuite extends DefaultTask {
+  private String solrImageName = null;
   private List<String> tests = new ArrayList<>();
   private List<String> ignore = new ArrayList<>();
 
   @OutputDirectory
   abstract DirectoryProperty getOutputDir()
 
+  public void setSolrImageName(String solrImageName) {
+    this.solrImageName = solrImageName
+  }
+
+  public String getSolrImageName() {
+    Preconditions.checkArgument(!Strings.isNullOrEmpty(solrImageName), "solrImageName is a required dockerTests configuration item.")
+    return solrImageName
+  }
+
   @Option(option = "tests", description = "Only run these specified tests, comma separated.")
   public void setTests(List<String> tests) {
     this.tests = tests;
@@ -81,6 +96,8 @@ abstract class DockerTestSuite extends DefaultTask {
 
   @TaskAction
   void execute() {
+    // Print information on the image before it is tested
+    project.logger.lifecycle("Testing Solr Image: $solrImageName\n")
     def sourceDir = project.file("tests/cases")
     sourceDir.eachFile  { file ->
       def testName = file.getName()
@@ -92,7 +109,7 @@ abstract class DockerTestSuite extends DefaultTask {
         project.exec {
           environment "TEST_DIR", "$file"
           environment "BUILD_DIR", "$testCaseBuildDir"
-          commandLine "bash", "$file/test.sh", "apache/solr:${project.version}"
+          commandLine "bash", "$file/test.sh", solrImageName
         }
       }
     }
@@ -101,4 +118,5 @@ abstract class DockerTestSuite extends DefaultTask {
 
 task testDocker(type: DockerTestSuite) {
   outputDir = project.file("$buildDir/tmp/tests")
+  solrImageName = dockerImageName
 }
\ No newline at end of file
diff --git a/solr/docker/tests/cases/version/test.sh b/solr/docker/tests/cases/version/test.sh
deleted file mode 100755
index 0f8c7ee..0000000
--- a/solr/docker/tests/cases/version/test.sh
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/bin/bash
-#
-set -euo pipefail
-
-TEST_DIR="${TEST_DIR:-$(dirname -- "${BASH_SOURCE[0]}")}"
-source "${TEST_DIR}/../../shared.sh"
-
-echo "Running $container_name"
-docker run --name "$container_name" -d "$tag"
-
-wait_for_server_started "$container_name"
-
-echo "Checking that the OS matches the tag '$tag'"
-if echo "$tag" | grep -q -- -alpine; then
-  alpine_version=$(docker exec --user=solr "$container_name" cat /etc/alpine-release || true)
-  if [[ -z $alpine_version ]]; then
-    echo "Could not get alpine version from container $container_name"
-    container_cleanup "$container_name"
-    exit 1
-  fi
-  echo "Alpine $alpine_version"
-else
-  debian_version=$(docker exec --user=solr "$container_name" cat /etc/debian_version || true)
-  if [[ -z $debian_version ]]; then
-    echo "Could not get debian version from container $container_name"
-    container_cleanup "$container_name"
-    exit 1
-  fi
-  echo "Debian $debian_version"
-fi
-
-# check that the version of Solr matches the tag
-changelog_version=$(docker exec --user=solr "$container_name" bash -c "grep -E '^==========* ' /opt/solr/CHANGES.txt | head -n 1 | tr -d '= '")
-echo "Solr version $changelog_version"
-solr_version_from_tag=$(echo "$tag" | sed -e 's/^.*://' -e 's/-.*//')
-
-if [[ $changelog_version != "$solr_version_from_tag" ]]; then
-  echo "Solr version mismatch"
-  container_cleanup "$container_name"
-  exit 1
-fi
-
-container_cleanup "$container_name"
-
-echo "Test $TEST_NAME $tag succeeded"