You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2022/03/08 06:47:43 UTC
[pulsar] branch master updated: [tests] Pulsar IO - Elasticsearch - reduce elastic container memory usage (#14580)
This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new a98b982 [tests] Pulsar IO - Elasticsearch - reduce elastic container memory usage (#14580)
a98b982 is described below
commit a98b98278be3fed966a2d42711921437a9cd47ba
Author: Nicolò Boschi <bo...@gmail.com>
AuthorDate: Tue Mar 8 07:45:36 2022 +0100
[tests] Pulsar IO - Elasticsearch - reduce elastic container memory usage (#14580)
---
.../io/elasticsearch/ElasticSearchBWCTests.java | 2 +-
.../elasticsearch/ElasticSearchClientSslTests.java | 16 ++++-----
.../io/elasticsearch/ElasticSearchClientTests.java | 17 +++++-----
.../elasticsearch/ElasticSearchExtractTests.java | 39 ++++++++--------------
.../ElasticSearchSinkRawDataTests.java | 11 +++---
.../io/elasticsearch/ElasticSearchSinkTests.java | 9 ++---
.../io/elasticsearch/ElasticSearchTestBase.java | 36 ++++++++++++++++++++
7 files changed, 72 insertions(+), 58 deletions(-)
diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchBWCTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchBWCTests.java
index 7b24a7c..86b9df3 100644
--- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchBWCTests.java
+++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchBWCTests.java
@@ -23,7 +23,7 @@ import org.apache.commons.lang3.tuple.Pair;
import org.apache.pulsar.client.api.schema.GenericObject;
import org.apache.pulsar.common.schema.SchemaType;
import org.apache.pulsar.functions.api.Record;
-import org.junit.Test;
+import org.testng.annotations.Test;
import java.nio.charset.StandardCharsets;
diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java
index 2070a46..3f2e499 100644
--- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java
+++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientSslTests.java
@@ -25,16 +25,12 @@ import org.testng.annotations.Test;
import java.io.IOException;
import java.time.Duration;
-import java.util.Optional;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
// see https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#ssl-tls-settings
-public class ElasticSearchClientSslTests {
-
- public static final String ELASTICSEARCH_IMAGE = Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE"))
- .orElse("docker.elastic.co/elasticsearch/elasticsearch:7.16.3-amd64");
+public class ElasticSearchClientSslTests extends ElasticSearchTestBase {
final static String INDEX = "myindex";
@@ -43,7 +39,7 @@ public class ElasticSearchClientSslTests {
@Test
public void testSslBasic() throws IOException {
- try(ElasticsearchContainer container = new ElasticsearchContainer(ELASTICSEARCH_IMAGE)
+ try (ElasticsearchContainer container = createElasticsearchContainer()
.withFileSystemBind(sslResourceDir, configDir + "/ssl")
.withPassword("elastic")
.withEnv("xpack.license.self_generated.type", "trial")
@@ -78,7 +74,7 @@ public class ElasticSearchClientSslTests {
@Test
public void testSslWithHostnameVerification() throws IOException {
- try(ElasticsearchContainer container = new ElasticsearchContainer(ELASTICSEARCH_IMAGE)
+ try (ElasticsearchContainer container = createElasticsearchContainer()
.withFileSystemBind(sslResourceDir, configDir + "/ssl")
.withPassword("elastic")
.withEnv("xpack.license.self_generated.type", "trial")
@@ -116,7 +112,7 @@ public class ElasticSearchClientSslTests {
@Test
public void testSslWithClientAuth() throws IOException {
- try(ElasticsearchContainer container = new ElasticsearchContainer(ELASTICSEARCH_IMAGE)
+ try(ElasticsearchContainer container = createElasticsearchContainer()
.withFileSystemBind(sslResourceDir, configDir + "/ssl")
.withPassword("elastic")
.withEnv("xpack.license.self_generated.type", "trial")
diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java
index 278a7cd..599a68d 100644
--- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java
+++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchClientTests.java
@@ -25,12 +25,12 @@ import org.apache.pulsar.client.api.schema.GenericObject;
import org.apache.pulsar.functions.api.Record;
import org.apache.pulsar.io.elasticsearch.testcontainers.ElasticToxiproxiContainer;
import org.awaitility.Awaitility;
-import org.junit.AfterClass;
import org.mockito.Mockito;
import org.opensearch.action.delete.DeleteRequest;
import org.opensearch.action.index.IndexRequest;
import org.testcontainers.containers.Network;
import org.testcontainers.elasticsearch.ElasticsearchContainer;
+import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -38,28 +38,29 @@ import java.io.IOException;
import java.util.Optional;
import java.util.UUID;
-import static org.junit.Assert.*;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.*;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertThrows;
+import static org.testng.Assert.assertTrue;
@Slf4j
-public class ElasticSearchClientTests {
- public static final String ELASTICSEARCH_IMAGE = Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE"))
- .orElse("docker.elastic.co/elasticsearch/elasticsearch:7.16.3-amd64");
+public class ElasticSearchClientTests extends ElasticSearchTestBase {
static ElasticsearchContainer container;
static Network network = Network.newNetwork();
@BeforeClass
public static final void initBeforeClass() throws IOException {
- container = new ElasticsearchContainer(ELASTICSEARCH_IMAGE).withNetwork(network);
+ container = createElasticsearchContainer().withNetwork(network);
container.start();
}
- @AfterClass
+ @AfterClass(alwaysRun = true)
public static void closeAfterClass() {
container.close();
network.close();
diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java
index 89d686c..b0d26d9 100644
--- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java
+++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java
@@ -18,8 +18,10 @@
*/
package org.apache.pulsar.io.elasticsearch;
-import com.google.common.collect.ImmutableList;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNull;
import com.google.common.collect.ImmutableMap;
+import java.util.Optional;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.client.api.schema.GenericObject;
@@ -31,33 +33,18 @@ import org.apache.pulsar.common.schema.KeyValue;
import org.apache.pulsar.common.schema.KeyValueEncodingType;
import org.apache.pulsar.common.schema.SchemaType;
import org.apache.pulsar.functions.api.Record;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
-import java.util.Collection;
-import java.util.Optional;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNull;
-
-@RunWith(Parameterized.class)
public class ElasticSearchExtractTests {
- SchemaType schemaType;
-
- @Parameters
- public static Collection schemaTypes() {
- return ImmutableList.of(SchemaType.JSON, SchemaType.AVRO);
- }
-
- public ElasticSearchExtractTests(SchemaType schemaType) {
- this.schemaType = schemaType;
+ @DataProvider(name = "schemaType")
+ public Object[] schemaType() {
+ return new Object[]{SchemaType.JSON, SchemaType.AVRO};
}
- @Test
- public void testGenericRecord() throws Exception {
+ @Test(dataProvider = "schemaType")
+ public void testGenericRecord(SchemaType schemaType) throws Exception {
RecordSchemaBuilder valueSchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("value");
valueSchemaBuilder.field("c").type(SchemaType.STRING).optional().defaultValue(null);
valueSchemaBuilder.field("d").type(SchemaType.INT32).optional().defaultValue(null);
@@ -154,8 +141,8 @@ public class ElasticSearchExtractTests {
assertNull(pair4.getRight());
}
- @Test
- public void testKeyValueGenericRecord() throws Exception {
+ @Test(dataProvider = "schemaType")
+ public void testKeyValueGenericRecord(SchemaType schemaType) throws Exception {
RecordSchemaBuilder keySchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("key");
keySchemaBuilder.field("a").type(SchemaType.STRING).optional().defaultValue(null);
keySchemaBuilder.field("b").type(SchemaType.INT32).optional().defaultValue(null);
@@ -273,4 +260,4 @@ public class ElasticSearchExtractTests {
assertEquals(pair3.getLeft(), "[\"1\",1]");
assertNull(pair3.getRight());
}
-}
+}
\ No newline at end of file
diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkRawDataTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkRawDataTests.java
index a4ecd64..018a11e 100644
--- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkRawDataTests.java
+++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkRawDataTests.java
@@ -23,11 +23,11 @@ import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.client.api.schema.GenericObject;
import org.apache.pulsar.functions.api.Record;
import org.apache.pulsar.io.core.SinkContext;
-import org.junit.AfterClass;
import org.mockito.Mock;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.testcontainers.elasticsearch.ElasticsearchContainer;
+import org.testng.annotations.AfterClass;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
@@ -43,10 +43,7 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-public class ElasticSearchSinkRawDataTests {
-
- public static final String ELASTICSEARCH_IMAGE = Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE"))
- .orElse("docker.elastic.co/elasticsearch/elasticsearch:7.16.3-amd64");
+public class ElasticSearchSinkRawDataTests extends ElasticSearchTestBase {
private static ElasticsearchContainer container;
@@ -65,11 +62,11 @@ public class ElasticSearchSinkRawDataTests {
@BeforeClass
public static final void initBeforeClass() {
- container = new ElasticsearchContainer(ELASTICSEARCH_IMAGE);
+ container = createElasticsearchContainer();
schema = Schema.BYTES;
}
- @AfterClass
+ @AfterClass(alwaysRun = true)
public static void closeAfterClass() {
container.close();
}
diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java
index 0377c76..86c39dd 100644
--- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java
+++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSinkTests.java
@@ -54,10 +54,7 @@ import org.testng.annotations.Test;
import java.util.Locale;
import static org.testng.Assert.assertNull;
-public class ElasticSearchSinkTests {
-
- public static final String ELASTICSEARCH_IMAGE = Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE"))
- .orElse("docker.elastic.co/elasticsearch/elasticsearch:7.16.3-amd64");
+public class ElasticSearchSinkTests extends ElasticSearchTestBase {
private static ElasticsearchContainer container;
@@ -76,7 +73,7 @@ public class ElasticSearchSinkTests {
@BeforeClass
public static final void initBeforeClass() {
- container = new ElasticsearchContainer(ELASTICSEARCH_IMAGE);
+ container = createElasticsearchContainer();
valueSchema = Schema.JSON(UserProfile.class);
genericSchema = Schema.generic(valueSchema.getSchemaInfo());
@@ -89,7 +86,7 @@ public class ElasticSearchSinkTests {
}
- @AfterClass
+ @AfterClass(alwaysRun = true)
public static void closeAfterClass() {
container.close();
}
diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchTestBase.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchTestBase.java
new file mode 100644
index 0000000..34e3fc2
--- /dev/null
+++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchTestBase.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.io.elasticsearch;
+
+import org.testcontainers.elasticsearch.ElasticsearchContainer;
+
+import java.util.Optional;
+
+public class ElasticSearchTestBase {
+
+ private static final String ELASTICSEARCH_IMAGE = Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE"))
+ .orElse("docker.elastic.co/elasticsearch/elasticsearch:7.16.3-amd64");
+
+ protected static ElasticsearchContainer createElasticsearchContainer() {
+ return new ElasticsearchContainer(ELASTICSEARCH_IMAGE)
+ .withEnv("ES_JAVA_OPTS", "-Xms128m -Xmx256m");
+
+ }
+
+}