You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by GitBox <gi...@apache.org> on 2018/10/21 01:19:00 UTC
[GitHub] gianm closed pull request #6497: include mysql-metadata-storage extension in distribution, but without…
gianm closed pull request #6497: include mysql-metadata-storage extension in distribution, but without…
URL: https://github.com/apache/incubator-druid/pull/6497
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/distribution/pom.xml b/distribution/pom.xml
index d9d7bc72b71..c7706cc5da8 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -164,25 +164,6 @@
</arguments>
</configuration>
</execution>
- <execution>
- <id>mysql-tarball</id>
- <phase>package</phase>
- <goals>
- <goal>exec</goal>
- </goals>
- <configuration>
- <executable>tar</executable>
- <arguments>
- <argument>-C</argument>
- <argument>${project.build.directory}/extensions</argument>
- <argument>-czvf</argument>
- <argument>
- ${project.build.directory}/mysql-metadata-storage-${project.parent.version}.tar.gz
- </argument>
- <argument>mysql-metadata-storage</argument>
- </arguments>
- </configuration>
- </execution>
</executions>
</plugin>
<plugin>
diff --git a/distribution/src/assembly/assembly.xml b/distribution/src/assembly/assembly.xml
index c1251f45a48..ab3e5e0d787 100644
--- a/distribution/src/assembly/assembly.xml
+++ b/distribution/src/assembly/assembly.xml
@@ -31,9 +31,6 @@
<includes>
<include>*/*</include>
</includes>
- <excludes>
- <exclude>mysql-metadata-storage/**</exclude>
- </excludes>
<outputDirectory>extensions</outputDirectory>
</fileSet>
diff --git a/docs/content/development/extensions-core/mysql.md b/docs/content/development/extensions-core/mysql.md
index a51b84ffa9d..09bec907128 100644
--- a/docs/content/development/extensions-core/mysql.md
+++ b/docs/content/development/extensions-core/mysql.md
@@ -6,6 +6,24 @@ layout: doc_page
Make sure to [include](../../operations/including-extensions.html) `mysql-metadata-storage` as an extension.
+<div class="note caution">
+The MySQL extension requires the MySQL Connector/J library which is not included in the Druid distribution.
+Refer to the following section for instructions on how to install this library.
+</div>
+
+## Installing the MySQL connector library
+
+This extension uses Oracle's MySQL JDBC driver which is not included in the Druid distribution and must be
+installed separately. There are a few ways to obtain this library:
+
+- It can be downloaded from the MySQL site at: https://dev.mysql.com/downloads/connector/j/
+- It can be fetched from Maven Central at: http://central.maven.org/maven2/mysql/mysql-connector-java/5.1.38/mysql-connector-java-5.1.38.jar
+- It may be available through your package manager, e.g. as `libmysql-java` on APT for a Debian-based OS
+
+This should fetch a JAR file named similar to 'mysql-connector-java-x.x.xx.jar'.
+
+Copy or symlink this file to `extensions/mysql-metadata-storage` under the distribution root directory.
+
## Setting up MySQL
1. Install MySQL
diff --git a/extensions-core/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml
index 2a8d5649b5d..bff092d0dd0 100644
--- a/extensions-core/mysql-metadata-storage/pom.xml
+++ b/extensions-core/mysql-metadata-storage/pom.xml
@@ -51,6 +51,7 @@
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.38</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.jdbi</groupId>
diff --git a/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnector.java b/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnector.java
index 8778eb3d37f..ffb9567be54 100644
--- a/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnector.java
+++ b/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnector.java
@@ -45,6 +45,7 @@
private static final String PAYLOAD_TYPE = "LONGBLOB";
private static final String SERIAL_TYPE = "BIGINT(20) AUTO_INCREMENT";
private static final String QUOTE_STRING = "`";
+ private static final String MYSQL_JDBC_DRIVER_CLASS_NAME = "com.mysql.jdbc.Driver";
private final DBI dbi;
@@ -57,11 +58,23 @@ public MySQLConnector(
{
super(config, dbTables);
+ try {
+ Class.forName(MYSQL_JDBC_DRIVER_CLASS_NAME, false, getClass().getClassLoader());
+ }
+ catch (ClassNotFoundException e) {
+ throw new ISE(e, "Could not find %s on the classpath. The MySQL Connector library is not included in the Druid "
+ + "distribution but is required to use MySQL. Please download a compatible library (for example "
+ + "'mysql-connector-java-5.1.38.jar') and place it under 'extensions/mysql-metadata-storage/'. See "
+ + "https://druid.apache.org/downloads for more details.",
+ MYSQL_JDBC_DRIVER_CLASS_NAME
+ );
+ }
+
final BasicDataSource datasource = getDatasource();
// MySQL driver is classloader isolated as part of the extension
// so we need to help JDBC find the driver
datasource.setDriverClassLoader(getClass().getClassLoader());
- datasource.setDriverClassName("com.mysql.jdbc.Driver");
+ datasource.setDriverClassName(MYSQL_JDBC_DRIVER_CLASS_NAME);
datasource.addConnectionProperty("useSSL", String.valueOf(connectorConfig.isUseSSL()));
if (connectorConfig.isUseSSL()) {
log.info("SSL is enabled on this MySQL connection. ");
diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile
index 5cbb54c4d8b..35deb9af44c 100644
--- a/integration-tests/docker/Dockerfile
+++ b/integration-tests/docker/Dockerfile
@@ -25,6 +25,11 @@ RUN find /var/lib/mysql -type f -exec touch {} \; && /etc/init.d/mysql start \
# Add Druid jars
ADD lib/* /usr/local/druid/lib/
+# Download the MySQL Java connector
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends apt-utils libmysql-java
+RUN ln -sf /usr/share/java/mysql-connector-java.jar /usr/local/druid/lib/mysql-connector-java.jar
+
# Add sample data
# touch is needed because OverlayFS's copy-up operation breaks POSIX standards. See https://github.com/docker/for-linux/issues/72.
RUN find /var/lib/mysql -type f -exec touch {} \; && service mysql start \
diff --git a/integration-tests/docker/broker.conf b/integration-tests/docker/broker.conf
index 4f49dfab418..39b8a7f94d4 100644
--- a/integration-tests/docker/broker.conf
+++ b/integration-tests/docker/broker.conf
@@ -57,7 +57,7 @@ command=java
-Ddruid.client.https.keyManagerPassword=druid123
-Ddruid.client.https.keyStorePassword=druid123
-Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
- -cp /shared/docker/lib/*
+ -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
org.apache.druid.cli.Main server broker
redirect_stderr=true
autorestart=false
diff --git a/integration-tests/docker/coordinator.conf b/integration-tests/docker/coordinator.conf
index 4fe113b745f..e05c924e947 100644
--- a/integration-tests/docker/coordinator.conf
+++ b/integration-tests/docker/coordinator.conf
@@ -51,7 +51,7 @@ command=java
-Ddruid.client.https.keyManagerPassword=druid123
-Ddruid.client.https.keyStorePassword=druid123
-Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
- -cp /shared/docker/lib/*
+ -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
org.apache.druid.cli.Main server coordinator
redirect_stderr=true
priority=100
diff --git a/integration-tests/docker/historical.conf b/integration-tests/docker/historical.conf
index 1c4fb96ae26..1b3a2ce5f29 100644
--- a/integration-tests/docker/historical.conf
+++ b/integration-tests/docker/historical.conf
@@ -53,7 +53,7 @@ command=java
-Ddruid.client.https.keyManagerPassword=druid123
-Ddruid.client.https.keyStorePassword=druid123
-Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
- -cp /shared/docker/lib/*
+ -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
org.apache.druid.cli.Main server historical
redirect_stderr=true
priority=100
diff --git a/integration-tests/docker/middlemanager.conf b/integration-tests/docker/middlemanager.conf
index 50985d0c0c2..efe4dec2e32 100644
--- a/integration-tests/docker/middlemanager.conf
+++ b/integration-tests/docker/middlemanager.conf
@@ -58,7 +58,7 @@ command=java
-Ddruid.client.https.keyStorePassword=druid123
-Ddruid.startup.logging.logProperties=true
-Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
- -cp /shared/docker/lib/*
+ -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
org.apache.druid.cli.Main server middleManager
redirect_stderr=true
priority=100
diff --git a/integration-tests/docker/overlord.conf b/integration-tests/docker/overlord.conf
index 304eb43482b..77514b1c253 100644
--- a/integration-tests/docker/overlord.conf
+++ b/integration-tests/docker/overlord.conf
@@ -52,7 +52,7 @@ command=java
-Ddruid.client.https.keyManagerPassword=druid123
-Ddruid.client.https.keyStorePassword=druid123
-Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
- -cp /shared/docker/lib/*
+ -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
org.apache.druid.cli.Main server overlord
redirect_stderr=true
priority=100
diff --git a/integration-tests/docker/router-no-client-auth-tls.conf b/integration-tests/docker/router-no-client-auth-tls.conf
index 804a3552b24..fb83b481984 100644
--- a/integration-tests/docker/router-no-client-auth-tls.conf
+++ b/integration-tests/docker/router-no-client-auth-tls.conf
@@ -47,7 +47,7 @@ command=java
-Ddruid.client.https.keyManagerPassword=druid123
-Ddruid.client.https.keyStorePassword=druid123
-Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
- -cp /shared/docker/lib/*
+ -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
org.apache.druid.cli.Main server router
redirect_stderr=true
priority=100
diff --git a/integration-tests/docker/router-permissive-tls.conf b/integration-tests/docker/router-permissive-tls.conf
index 1453c4b02e9..62932c7a942 100644
--- a/integration-tests/docker/router-permissive-tls.conf
+++ b/integration-tests/docker/router-permissive-tls.conf
@@ -47,7 +47,7 @@ command=java
-Ddruid.client.https.keyManagerPassword=druid123
-Ddruid.client.https.keyStorePassword=druid123
-Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
- -cp /shared/docker/lib/*
+ -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
org.apache.druid.cli.Main server router
redirect_stderr=true
priority=100
diff --git a/integration-tests/docker/router.conf b/integration-tests/docker/router.conf
index a4a604ad7ab..29bb6b52385 100644
--- a/integration-tests/docker/router.conf
+++ b/integration-tests/docker/router.conf
@@ -46,7 +46,7 @@ command=java
-Ddruid.client.https.keyManagerPassword=druid123
-Ddruid.client.https.keyStorePassword=druid123
-Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
- -cp /shared/docker/lib/*
+ -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
org.apache.druid.cli.Main server router
redirect_stderr=true
priority=100
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org