You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metamodel.apache.org by ka...@apache.org on 2018/06/06 03:44:16 UTC
[3/8] metamodel git commit: Toggled Kafka integration tests using the
properties file
Toggled Kafka integration tests using the properties file
Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo
Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/26641cc4
Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/26641cc4
Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/26641cc4
Branch: refs/heads/master
Commit: 26641cc435db46646e5ce5732df09c58781b4ce8
Parents: c6be1ae
Author: Kasper Sørensen <i....@gmail.com>
Authored: Sun Jan 28 15:05:43 2018 -0800
Committer: Kasper Sørensen <i....@gmail.com>
Committed: Sun Jan 28 15:05:43 2018 -0800
----------------------------------------------------------------------
...del-integrationtest-configuration.properties | 7 ++
.../kafka/KafkaDataContextIntegrationTest.java | 36 ++++++----
.../apache/metamodel/kafka/KafkaTestServer.java | 74 ++++++++++++++++++++
...del-integrationtest-configuration.properties | 7 ++
4 files changed, 112 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metamodel/blob/26641cc4/example-metamodel-integrationtest-configuration.properties
----------------------------------------------------------------------
diff --git a/example-metamodel-integrationtest-configuration.properties b/example-metamodel-integrationtest-configuration.properties
index 4f3cbfe..20bba26 100644
--- a/example-metamodel-integrationtest-configuration.properties
+++ b/example-metamodel-integrationtest-configuration.properties
@@ -45,6 +45,13 @@
#hbase.zookeeper.hostname=localhost
#hbase.zookeeper.port=2181
+# ------------------------
+# Kafka module properties:
+# ------------------------
+
+#kafka.bootstrap.servers=localhost:9092
+#kafka.topic.prefix=metamodel_test
+
# -----------------------
# JDBC module properties:
# -----------------------
http://git-wip-us.apache.org/repos/asf/metamodel/blob/26641cc4/kafka/src/test/java/org/apache/metamodel/kafka/KafkaDataContextIntegrationTest.java
----------------------------------------------------------------------
diff --git a/kafka/src/test/java/org/apache/metamodel/kafka/KafkaDataContextIntegrationTest.java b/kafka/src/test/java/org/apache/metamodel/kafka/KafkaDataContextIntegrationTest.java
index 2d0cf5e..ffc4075 100644
--- a/kafka/src/test/java/org/apache/metamodel/kafka/KafkaDataContextIntegrationTest.java
+++ b/kafka/src/test/java/org/apache/metamodel/kafka/KafkaDataContextIntegrationTest.java
@@ -17,34 +17,46 @@ import org.apache.metamodel.DataContext;
import org.apache.metamodel.data.DataSet;
import org.apache.metamodel.data.WrappingDataSet;
import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class KafkaDataContextIntegrationTest {
- private static final String BOOTSTRAP_SERVERS = "localhost:9092";
private static final Logger logger = LoggerFactory.getLogger(KafkaDataContextIntegrationTest.class);
+ private final KafkaTestServer testServer = new KafkaTestServer();
+
+ @Before
+ public void setUp() {
+ final boolean configured = testServer.isConfigured();
+ if (!configured) {
+ System.err.println(testServer.getInvalidConfigurationMessage());
+ }
+ Assume.assumeTrue(configured);
+ }
+
@Test
public void testGetSchemaInfo() {
- final DataContext dataContext1 = new KafkaDataContext<>(String.class, String.class, BOOTSTRAP_SERVERS, Arrays
- .asList("non-existing-topic"));
+ final DataContext dataContext1 = new KafkaDataContext<>(String.class, String.class, testServer
+ .getBootstrapServers(), Arrays.asList("non-existing-topic"));
Assert.assertEquals("[non-existing-topic, default_table]", dataContext1.getDefaultSchema().getTableNames()
.toString());
- final DataContext dataContext2 = new KafkaDataContext<>(String.class, String.class, BOOTSTRAP_SERVERS, Arrays
- .asList("test1", "test2", "test3"));
+ final DataContext dataContext2 = new KafkaDataContext<>(String.class, String.class, testServer
+ .getBootstrapServers(), Arrays.asList("test1", "test2", "test3"));
Assert.assertEquals("[test1, test2, test3]", dataContext2.getDefaultSchema().getTableNames().toString());
}
@Test
public void testQueryNoFilters() {
- final String topic = "test_" + UUID.randomUUID().toString().replaceAll("\\-", "");
+ final String topic = testServer.getTopicPrefix() + UUID.randomUUID().toString().replaceAll("\\-", "");
- final DataContext dataContext = new KafkaDataContext<>(String.class, String.class, BOOTSTRAP_SERVERS, Arrays
- .asList(topic));
+ final DataContext dataContext = new KafkaDataContext<>(String.class, String.class, testServer
+ .getBootstrapServers(), Arrays.asList(topic));
Assert.assertEquals("[" + topic + ", default_table]", dataContext.getDefaultSchema().getTableNames()
.toString());
@@ -72,10 +84,10 @@ public class KafkaDataContextIntegrationTest {
@Test
public void testQueryUsingOffset() throws InterruptedException {
- final String topic = "test_" + UUID.randomUUID().toString().replaceAll("\\-", "");
+ final String topic = testServer.getTopicPrefix() + UUID.randomUUID().toString().replaceAll("\\-", "");
- final DataContext dataContext = new KafkaDataContext<>(String.class, String.class, BOOTSTRAP_SERVERS, Arrays
- .asList(topic));
+ final DataContext dataContext = new KafkaDataContext<>(String.class, String.class, testServer
+ .getBootstrapServers(), Arrays.asList(topic));
final int numRecords = 1000;
final int queriedOffset = 500;
@@ -117,7 +129,7 @@ public class KafkaDataContextIntegrationTest {
@Override
public void run() {
final Properties properties = new Properties();
- properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS);
+ properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, testServer.getBootstrapServers());
properties.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
properties.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
properties.setProperty(ProducerConfig.CLIENT_ID_CONFIG, "metamodel-test");
http://git-wip-us.apache.org/repos/asf/metamodel/blob/26641cc4/kafka/src/test/java/org/apache/metamodel/kafka/KafkaTestServer.java
----------------------------------------------------------------------
diff --git a/kafka/src/test/java/org/apache/metamodel/kafka/KafkaTestServer.java b/kafka/src/test/java/org/apache/metamodel/kafka/KafkaTestServer.java
new file mode 100644
index 0000000..28a4754
--- /dev/null
+++ b/kafka/src/test/java/org/apache/metamodel/kafka/KafkaTestServer.java
@@ -0,0 +1,74 @@
+package org.apache.metamodel.kafka;
+
+import java.io.File;
+import java.io.FileReader;
+import java.util.Properties;
+
+public class KafkaTestServer {
+
+ private final boolean _configured;
+ private String _bootstrapServers;
+ private String _topicPrefix;
+
+ public KafkaTestServer() {
+ final File file = new File(getPropertyFilePath());
+ if (file.exists()) {
+ _configured = loadPropertyFile(file);
+ } else {
+ // Continuous integration case
+ if (System.getenv("CONTINUOUS_INTEGRATION") != null) {
+ File travisFile = new File("../travis-metamodel-integrationtest-configuration.properties");
+ if (travisFile.exists()) {
+ _configured = loadPropertyFile(travisFile);
+ } else {
+ _configured = false;
+ }
+ } else {
+ _configured = false;
+ }
+ }
+ }
+
+ public boolean isConfigured() {
+ return _configured;
+ }
+
+ public String getBootstrapServers() {
+ return _bootstrapServers;
+ }
+
+ public String getTopicPrefix() {
+ return _topicPrefix;
+ }
+
+ private boolean loadPropertyFile(File file) {
+ final Properties properties = new Properties();
+ try {
+ properties.load(new FileReader(file));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ _bootstrapServers = properties.getProperty("kafka.bootstrap.servers");
+ _topicPrefix = properties.getProperty("kafka.topic.prefix");
+
+ final boolean configured = (_bootstrapServers != null && !_bootstrapServers.isEmpty() && _topicPrefix != null
+ && !_topicPrefix.isEmpty());
+
+ if (configured) {
+ System.out.println("Loaded Kafka configuration. BootstrapServers=" + _bootstrapServers + ", TopicPrefix="
+ + _topicPrefix);
+ }
+ return configured;
+ }
+
+ private String getPropertyFilePath() {
+ String userHome = System.getProperty("user.home");
+ return userHome + "/metamodel-integrationtest-configuration.properties";
+ }
+
+ public String getInvalidConfigurationMessage() {
+ return "!!! WARN !!! Kafka module ignored\r\n" + "Please configure kafka connection locally ("
+ + getPropertyFilePath() + "), to run integration tests";
+ }
+}
http://git-wip-us.apache.org/repos/asf/metamodel/blob/26641cc4/travis-metamodel-integrationtest-configuration.properties
----------------------------------------------------------------------
diff --git a/travis-metamodel-integrationtest-configuration.properties b/travis-metamodel-integrationtest-configuration.properties
index f551fef..a336c0f 100644
--- a/travis-metamodel-integrationtest-configuration.properties
+++ b/travis-metamodel-integrationtest-configuration.properties
@@ -27,6 +27,13 @@ mongodb.collectionName=my_collection
#hbase.zookeeper.port=2181
# ------------------------
+# Kafka module properties:
+# ------------------------
+
+#kafka.bootstrap.servers=localhost:9092
+#kafka.topic.prefix=metamodel_test
+
+# ------------------------
# Neo4j module properties:
# ------------------------