You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vi...@apache.org on 2019/05/31 05:46:29 UTC
[incubator-hudi] branch master updated: Create
hoodie-utilities-bundle to host the shaded jar
This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push:
new acd7412 Create hoodie-utilities-bundle to host the shaded jar
acd7412 is described below
commit acd74129cd97f24c0dde9bf032a4048f2ce27b5f
Author: Vinoth Chandar <vi...@uber.com>
AuthorDate: Thu May 30 20:06:26 2019 -0700
Create hoodie-utilities-bundle to host the shaded jar
- hoodie-utilities can now be pulled in as compile time dependency
- Lets users test their DeltaStreamer transformers for e.g
- Tested the docker demo works & takes in the bundle
- Doc changes to follow, to move DeltaStreamer commands to bundle jar
---
docker/hoodie/hadoop/hive_base/pom.xml | 4 +-
hoodie-utilities/pom.xml | 125 ---------------------
.../hoodie-utilities-bundle}/pom.xml | 34 ++++--
.../com/uber/hoodie/utilities/bundle/Main.java | 36 ++++++
pom.xml | 1 +
5 files changed, 64 insertions(+), 136 deletions(-)
diff --git a/docker/hoodie/hadoop/hive_base/pom.xml b/docker/hoodie/hadoop/hive_base/pom.xml
index edf68ee..57199ad 100644
--- a/docker/hoodie/hadoop/hive_base/pom.xml
+++ b/docker/hoodie/hadoop/hive_base/pom.xml
@@ -55,7 +55,9 @@
<copy file="${project.basedir}/../../../../packaging/hoodie-hadoop-mr-bundle/target/hoodie-hadoop-mr-bundle-${project.version}.jar" tofile="target/hoodie-hadoop-mr-bundle.jar" />
<copy file="${project.basedir}/../../../../packaging/hoodie-hive-bundle/target/hoodie-hive-bundle-${project.version}.jar" tofile="target/hoodie-hive-bundle.jar" />
<copy file="${project.basedir}/../../../../packaging/hoodie-spark-bundle/target/hoodie-spark-bundle-${project.version}.jar" tofile="target/hoodie-spark-bundle.jar" />
- <copy file="${project.basedir}/../../../../hoodie-utilities/target/hoodie-utilities-${project.version}.jar" tofile="target/hoodie-utilities.jar" />
+ <copy
+ file="${project.basedir}/../../../../packaging/hoodie-utilities-bundle/target/hoodie-utilities-bundle-${project.version}.jar"
+ tofile="target/hoodie-utilities.jar"/>
</tasks>
</configuration>
<goals>
diff --git a/hoodie-utilities/pom.xml b/hoodie-utilities/pom.xml
index b3218fa..6fb720f 100644
--- a/hoodie-utilities/pom.xml
+++ b/hoodie-utilities/pom.xml
@@ -40,131 +40,6 @@
<target>1.8</target>
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>3.1.1</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <dependencyReducedPomLocation>${project.build.directory}/dependency-reduced-pom.xml
- </dependencyReducedPomLocation>
- <artifactSet>
- <includes>
- <include>commons-codec:commons-codec</include>
- <include>commons-dbcp:commons-dbcp</include>
- <include>commons-pool:commons-pool</include>
- <include>com.uber.hoodie:hoodie-common</include>
- <include>com.uber.hoodie:hoodie-client</include>
- <include>com.uber.hoodie:hoodie-spark</include>
- <include>com.uber.hoodie:hoodie-hive</include>
- <include>com.uber.hoodie:hoodie-hadoop-mr</include>
- <include>com.uber.hoodie:hoodie-timeline-service</include>
- <include>com.beust:jcommander</include>
- <include>com.twitter:bijection-avro_2.11</include>
- <include>com.twitter:bijection-core_2.11</include>
- <include>org.apache.parquet:parquet-avro</include>
- <include>com.twitter:parquet-avro</include>
- <include>com.twitter.common:objectsize</include>
- <include>io.confluent:kafka-avro-serializer</include>
- <include>io.confluent:common-config</include>
- <include>io.confluent:common-utils</include>
- <include>io.confluent:kafka-schema-registry-client</include>
- <include>io.dropwizard.metrics:metrics-core</include>
- <include>io.dropwizard.metrics:metrics-graphite</include>
- <include>org.apache.spark:spark-streaming-kafka-0-8_2.11</include>
- <include>org.apache.kafka:kafka_2.11</include>
- <include>com.101tec:zkclient</include>
- <include>org.apache.kafka:kafka-clients</include>
- <include>org.apache.hive:hive-common</include>
- <include>org.apache.hive:hive-service</include>
- <include>org.apache.hive:hive-metastore</include>
- <include>org.apache.hive:hive-jdbc</include>
- <include>com.esotericsoftware:kryo-shaded</include>
- <include>org.objenesis:objenesis</include>
- <include>com.esotericsoftware:minlog</include>
- <include>com.yammer.metrics:metrics-core</include>
- </includes>
- </artifactSet>
- <relocations>
- <relocation>
- <pattern>com.beust.jcommander.</pattern>
- <shadedPattern>com.uber.hoodie.com.beust.jcommander.</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.commons.dbcp.</pattern>
- <shadedPattern>com.uber.hoodie.org.apache.commons.dbcp.</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.commons.pool.</pattern>
- <shadedPattern>com.uber.hoodie.org.apache.commons.pool.</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.hive.jdbc.</pattern>
- <shadedPattern>com.uber.hoodie.org.apache.hive.jdbc.</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.hadoop.hive.metastore.</pattern>
- <shadedPattern>com.uber.hoodie.org.apache.hadoop_hive.metastore.</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.hive.common.</pattern>
- <shadedPattern>com.uber.hoodie.org.apache.hive.common.</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.hadoop.hive.common.</pattern>
- <shadedPattern>com.uber.hoodie.org.apache.hadoop_hive.common.</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.hadoop.hive.conf.</pattern>
- <shadedPattern>com.uber.hoodie.org.apache.hadoop_hive.conf.</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.hive.service.</pattern>
- <shadedPattern>com.uber.hoodie.org.apache.hive.service.</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.hadoop.hive.service.</pattern>
- <shadedPattern>com.uber.hoodie.org.apache.hadoop_hive.service.</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.esotericsoftware.kryo.</pattern>
- <shadedPattern>com.uber.hoodie.com.esotericsoftware.kryo.</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.objenesis.</pattern>
- <shadedPattern>com.uber.hoodie.org.objenesis.</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.esotericsoftware.minlog.</pattern>
- <shadedPattern>com.uber.hoodie.com.esotericsoftware.minlog.</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.codahale.metrics.</pattern>
- <shadedPattern>com.uber.hoodie.com.codahale.metrics.</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.commons.codec.</pattern>
- <shadedPattern>com.uber.hoodie.org.apache.commons.codec.</shadedPattern>
- </relocation>
- </relocations>
- <filters>
- <filter>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
<resources>
diff --git a/hoodie-utilities/pom.xml b/packaging/hoodie-utilities-bundle/pom.xml
similarity index 93%
copy from hoodie-utilities/pom.xml
copy to packaging/hoodie-utilities-bundle/pom.xml
index b3218fa..a1ac2c0 100644
--- a/hoodie-utilities/pom.xml
+++ b/packaging/hoodie-utilities-bundle/pom.xml
@@ -1,12 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
- ~ Copyright (c) 2016 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
+ ~ Copyright (c) 2019 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
~
- ~ Licensed under the Apache License, Version 2.0 (the "License");
- ~ you may not use this file except in compliance with the License.
- ~ You may obtain a copy of the License at
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
~
- ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,17 +15,24 @@
~ limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>hoodie</artifactId>
<groupId>com.uber.hoodie</groupId>
<version>0.4.8-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
-
- <artifactId>hoodie-utilities</artifactId>
+ <artifactId>hoodie-utilities-bundle</artifactId>
<packaging>jar</packaging>
+ <properties>
+ <log4j.version>1.2.17</log4j.version>
+ <junit.version>4.10</junit.version>
+ <checkstyle.skip>true</checkstyle.skip>
+ </properties>
+
<build>
<plugins>
<plugin>
@@ -60,6 +67,7 @@
<include>commons-pool:commons-pool</include>
<include>com.uber.hoodie:hoodie-common</include>
<include>com.uber.hoodie:hoodie-client</include>
+ <include>com.uber.hoodie:hoodie-utilities</include>
<include>com.uber.hoodie:hoodie-spark</include>
<include>com.uber.hoodie:hoodie-hive</include>
<include>com.uber.hoodie:hoodie-hadoop-mr</include>
@@ -300,6 +308,12 @@
<dependency>
<groupId>com.uber.hoodie</groupId>
+ <artifactId>hoodie-utilities</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.uber.hoodie</groupId>
<artifactId>hoodie-client</artifactId>
<version>${project.version}</version>
<classifier>tests</classifier>
@@ -348,7 +362,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
- </dependency>
+ </dependency>
<dependency>
<groupId>org.apache.spark</groupId>
@@ -442,6 +456,6 @@
<artifactId>kafka-schema-registry-client</artifactId>
<version>3.0.0</version>
</dependency>
-
</dependencies>
</project>
+
diff --git a/packaging/hoodie-utilities-bundle/src/main/java/com/uber/hoodie/utilities/bundle/Main.java b/packaging/hoodie-utilities-bundle/src/main/java/com/uber/hoodie/utilities/bundle/Main.java
new file mode 100644
index 0000000..e1ed070
--- /dev/null
+++ b/packaging/hoodie-utilities-bundle/src/main/java/com/uber/hoodie/utilities/bundle/Main.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2019 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ */
+
+package com.uber.hoodie.utilities.bundle;
+
+import com.uber.hoodie.common.util.ReflectionUtils;
+
+/**
+ * A simple main class to dump all classes loaded in current classpath
+ *
+ * This is a workaround for generating sources and javadoc jars for packaging modules. The maven plugins for generating
+ * javadoc and sources plugins do not generate corresponding jars if there are no source files.
+ *
+ * This class does not have anything to do with Hudi but is there to keep mvn javadocs/source plugin happy.
+ */
+public class Main {
+
+ public static void main(String[] args) {
+ ReflectionUtils.getTopLevelClassesInClasspath(Main.class).forEach(System.out::println);
+ }
+}
diff --git a/pom.xml b/pom.xml
index 05d4895..1e9e018 100644
--- a/pom.xml
+++ b/pom.xml
@@ -41,6 +41,7 @@
<module>packaging/hoodie-hive-bundle</module>
<module>packaging/hoodie-spark-bundle</module>
<module>packaging/hoodie-presto-bundle</module>
+ <module>packaging/hoodie-utilities-bundle</module>
<module>docker/hoodie/hadoop</module>
<module>hoodie-integ-test</module>
</modules>