You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by fo...@apache.org on 2022/09/19 07:40:39 UTC
[jackrabbit-oak] branch trunk updated: OAK-9932 - Allow specifying the version of the ES docker image used for tests. (#705)
This is an automated email from the ASF dual-hosted git repository.
fortino pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new b84f904fa8 OAK-9932 - Allow specifying the version of the ES docker image used for tests. (#705)
b84f904fa8 is described below
commit b84f904fa83bf2129697a2658b7b232008a27bc9
Author: Nuno Santos <ns...@adobe.com>
AuthorDate: Mon Sep 19 09:40:32 2022 +0200
OAK-9932 - Allow specifying the version of the ES docker image used for tests. (#705)
* Allow specifying the version of the ES docker image used for tests.
* Add license information to new file.
* Set maximum heap size of Elasticsearch running inside the test container to 1GB.
* Fake commit to force retest.
* Save output of ES test container to a local log file.
* Redirect output of ES container into the standard logger.
---
.../plugins/index/elastic/ElasticTestServer.java | 101 ++++++++++++++-------
.../plugins/index/elastic/ElasticTestUtils.java | 4 +
...ticstartscript.sh => elasticsearch-plugins.yml} | 7 +-
.../src/test/resources/elasticsearch.yml | 7 +-
4 files changed, 81 insertions(+), 38 deletions(-)
diff --git a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticTestServer.java b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticTestServer.java
index 5a7b6eece5..c0ef45a669 100644
--- a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticTestServer.java
+++ b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticTestServer.java
@@ -18,15 +18,13 @@ package org.apache.jackrabbit.oak.plugins.index.elastic;
import co.elastic.clients.transport.Version;
import com.github.dockerjava.api.DockerClient;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
+import com.google.common.collect.ImmutableMap;
import org.apache.jackrabbit.oak.commons.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.Network;
+import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.elasticsearch.ElasticsearchContainer;
import org.testcontainers.utility.MountableFile;
@@ -34,16 +32,22 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.net.URL;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
+import java.util.Map;
import static org.junit.Assume.assumeNotNull;
public class ElasticTestServer implements AutoCloseable {
private static final Logger LOG = LoggerFactory.getLogger(ElasticTestServer.class);
- private static final String PLUGIN_DIGEST = "326893bb98ef1a0c569d9f4c4a9a073e53361924f990b17e87077985ce8a7478";
+ private static final Map<String, String> PLUGIN_OFFICIAL_RELEASES_DIGEST_MAP = ImmutableMap.of(
+ "7.17.3.0", "5e3b40bb72b2813f927be9bf6ecdf88668d89d2ef20c7ebafaa51ab8407fd179",
+ "7.17.6.0", "326893bb98ef1a0c569d9f4c4a9a073e53361924f990b17e87077985ce8a7478"
+ );
+
private static final ElasticTestServer SERVER = new ElasticTestServer();
private static volatile ElasticsearchContainer CONTAINER;
@@ -70,20 +74,35 @@ public class ElasticTestServer implements AutoCloseable {
}
private synchronized void setup() {
- final String pluginVersion = Version.VERSION + ".0";
+ String esDockerImageVersion = ElasticTestUtils.ELASTIC_DOCKER_IMAGE_VERSION;
+ if (esDockerImageVersion == null) {
+ esDockerImageVersion = Version.VERSION.toString();
+ }
+ final String pluginVersion = esDockerImageVersion + ".0";
final String pluginFileName = "elastiknn-" + pluginVersion + ".zip";
final String localPluginPath = "target/" + pluginFileName;
+ LOG.info("Elasticsearch test Docker image version: {}.", esDockerImageVersion);
downloadSimilaritySearchPluginIfNotExists(localPluginPath, pluginVersion);
checkIfDockerClientAvailable();
Network network = Network.newNetwork();
- CONTAINER = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:" + Version.VERSION)
- .withCopyFileToContainer(MountableFile.forClasspathResource("elasticsearch.yml"), "/usr/share/elasticsearch/config/")
- .withCopyFileToContainer(MountableFile.forHostPath(localPluginPath), "/tmp/plugins/" + pluginFileName)
- .withCopyFileToContainer(MountableFile.forClasspathResource("elasticstartscript.sh"), "/tmp/elasticstartscript.sh")
- .withCommand("bash /tmp/elasticstartscript.sh")
+ CONTAINER = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:" + esDockerImageVersion)
+ .withEnv("ES_JAVA_OPTS", "-Xms1g -Xmx1g")
+ .withCopyFileToContainer(
+ MountableFile.forClasspathResource("elasticsearch.yml"),
+ "/usr/share/elasticsearch/config/elasticsearch.yml")
+ // https://www.elastic.co/guide/en/elasticsearch/plugins/8.4/manage-plugins-using-configuration-file.html
+ .withCopyFileToContainer(
+ MountableFile.forClasspathResource("elasticsearch-plugins.yml"),
+ "/usr/share/elasticsearch/config/elasticsearch-plugins.yml")
+ .withCopyFileToContainer(
+ MountableFile.forHostPath(localPluginPath),
+ "/tmp/plugins/elastiknn.zip")
.withNetwork(network)
.withStartupAttempts(3);
CONTAINER.start();
+
+ Slf4jLogConsumer logConsumer = new Slf4jLogConsumer(LOG).withSeparateOutputStreams();
+ CONTAINER.followOutput(logConsumer);
}
@Override
@@ -103,27 +122,44 @@ public class ElasticTestServer implements AutoCloseable {
File pluginFile = new File(localPluginPath);
if (!pluginFile.exists()) {
LOG.info("Plugin file {} doesn't exist. Trying to download.", localPluginPath);
- try (CloseableHttpClient client = HttpClients.createDefault()) {
- HttpGet get = new HttpGet("https://github.com/alexklibisz/elastiknn/releases/download/" + pluginVersion
- + "/elastiknn-" + pluginVersion + ".zip");
- CloseableHttpResponse response = client.execute(get);
- InputStream inputStream = response.getEntity().getContent();
- MessageDigest messageDigest = MessageDigest.getInstance("SHA-256");
- DigestInputStream dis = new DigestInputStream(inputStream, messageDigest);
- FileOutputStream outputStream = new FileOutputStream(pluginFile);
- IOUtils.copy(dis, outputStream);
- messageDigest = dis.getMessageDigest();
- // bytes to hex
- StringBuilder result = new StringBuilder();
- for (byte b : messageDigest.digest()) {
- result.append(String.format("%02x", b));
+ String pluginUri;
+ String pluginDigest;
+ if (PLUGIN_OFFICIAL_RELEASES_DIGEST_MAP.containsKey(pluginVersion)) {
+ pluginDigest = PLUGIN_OFFICIAL_RELEASES_DIGEST_MAP.get(pluginVersion);
+ pluginUri = "https://github.com/alexklibisz/elastiknn/releases/download/" + pluginVersion
+ + "/elastiknn-" + pluginVersion + ".zip";
+ } else {
+ pluginDigest = null; // Skip validation
+ pluginUri = ElasticTestUtils.ELASTIC_KNN_PLUGIN_URI;
+ if (pluginUri == null) {
+ throw new RuntimeException("Elastiknn " + pluginVersion + " is not a known official release, so it cannot be downloaded from the official GitHub repo. Please provide the download URI in system property \"" + ElasticTestUtils.ELASTIC_KNN_PLUGIN_URI_KEY + "\".");
}
- if (!PLUGIN_DIGEST.equals(result.toString())) {
- String deleteString = "Downloaded plugin file deleted.";
- if (!pluginFile.delete()) {
- deleteString = "Could not delete downloaded plugin file.";
+ }
+ LOG.info("Downloading Elastiknn plugin from {}.", pluginUri);
+ try {
+ try (InputStream inputStream = new URL(pluginUri).openStream();
+ FileOutputStream outputStream = new FileOutputStream(pluginFile)
+ ) {
+ if (pluginDigest != null) {
+ MessageDigest messageDigest = MessageDigest.getInstance("SHA-256");
+ DigestInputStream dis = new DigestInputStream(inputStream, messageDigest);
+ IOUtils.copy(dis, outputStream);
+ messageDigest = dis.getMessageDigest();
+ // bytes to hex
+ StringBuilder result = new StringBuilder();
+ for (byte b : messageDigest.digest()) {
+ result.append(String.format("%02x", b));
+ }
+ if (!pluginDigest.equals(result.toString())) {
+ String deleteString = "Downloaded plugin file deleted.";
+ if (!pluginFile.delete()) {
+ deleteString = "Could not delete downloaded plugin file.";
+ }
+ throw new RuntimeException("Plugin digest unequal. Found " + result + ". Expected " + pluginDigest + ". " + deleteString);
+ }
+ } else {
+ IOUtils.copy(inputStream, outputStream);
}
- throw new RuntimeException("Plugin digest unequal. Found " + result + ". Expected " + PLUGIN_DIGEST + ". " + deleteString);
}
} catch (IOException | NoSuchAlgorithmException e) {
throw new RuntimeException("Could not download similarity search plugin", e);
@@ -147,10 +183,9 @@ public class ElasticTestServer implements AutoCloseable {
*/
public static void main(String[] args) throws IOException {
ElasticsearchContainer esContainer = ElasticTestServer.getESTestServer();
- System.out.println("Docker container with Elasticsearch launched at \""+esContainer.getHttpHostAddress()+
- "\". Please PRESS ENTER to stop it...");
+ System.out.println("Docker container with Elasticsearch launched at \"" + esContainer.getHttpHostAddress() +
+ "\". Please PRESS ENTER to stop it...");
System.in.read();
esContainer.stop();
}
-
}
diff --git a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticTestUtils.java b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticTestUtils.java
index 62eb4f5484..a0ce969f67 100644
--- a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticTestUtils.java
+++ b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticTestUtils.java
@@ -31,6 +31,10 @@ public final class ElasticTestUtils {
// Do not set this if docker is running and you want to run the tests on docker instead.
public static final String ELASTIC_CONNECTION_STRING = System.getProperty("elasticConnectionString");
+ public static final String ELASTIC_DOCKER_IMAGE_VERSION = System.getProperty("elasticDockerImageVersion");
+ public static final String ELASTIC_KNN_PLUGIN_URI_KEY = "elasticKnnPluginUri";
+ public static final String ELASTIC_KNN_PLUGIN_URI = System.getProperty(ELASTIC_KNN_PLUGIN_URI_KEY);
+
public static void assertEventually(Runnable r, long timeoutMillis) {
final long start = System.currentTimeMillis();
long lastAttempt = 0;
diff --git a/oak-search-elastic/src/test/resources/elasticstartscript.sh b/oak-search-elastic/src/test/resources/elasticsearch-plugins.yml
similarity index 77%
rename from oak-search-elastic/src/test/resources/elasticstartscript.sh
rename to oak-search-elastic/src/test/resources/elasticsearch-plugins.yml
index 2b54487ace..96841ed0a3 100644
--- a/oak-search-elastic/src/test/resources/elasticstartscript.sh
+++ b/oak-search-elastic/src/test/resources/elasticsearch-plugins.yml
@@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-pluginZip=`ls /tmp/plugins | grep elastiknn-7.17 | head -n 1`
-echo "Installing plugin /tmp/plugins/$pluginZip"
-bin/elasticsearch-plugin install --batch file:///tmp/plugins/$pluginZip
-su -c "bin/elasticsearch" elasticsearch
\ No newline at end of file
+plugins:
+ - id: elastiknn
+ location: file:///tmp/plugins/elastiknn.zip
\ No newline at end of file
diff --git a/oak-search-elastic/src/test/resources/elasticsearch.yml b/oak-search-elastic/src/test/resources/elasticsearch.yml
index acd8502053..c5b2dc7588 100644
--- a/oak-search-elastic/src/test/resources/elasticsearch.yml
+++ b/oak-search-elastic/src/test/resources/elasticsearch.yml
@@ -16,4 +16,9 @@
#
network.host: 0.0.0.0
ingest.geoip.downloader.enabled: false
-xpack.security.enabled: false
\ No newline at end of file
+xpack.security.enabled: false
+
+# In ES 8.0, by default it is no longer possible to use wildcards to delete several indexes in a single operation.
+# This is used by the tests for cleanup, so we must set this to true explicitly.
+# https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html
+action.destructive_requires_name: false
\ No newline at end of file