You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by jw...@apache.org on 2019/11/14 21:54:23 UTC
[crunch] 01/03: WIP for modernizing Crunch deps
This is an automated email from the ASF dual-hosted git repository.
jwills pushed a commit to branch jwills_great_version_upgrade
in repository https://gitbox.apache.org/repos/asf/crunch.git
commit d94800c556368829870a9bac64618dfe65d5699d
Author: Josh Wills <jw...@apache.org>
AuthorDate: Wed Nov 13 15:14:51 2019 -0800
WIP for modernizing Crunch deps
---
crunch-core/pom.xml | 9 +++--
.../io/parquet/AvroParquetFileSourceTargetIT.java | 2 +-
.../crunch/io/parquet/AvroParquetFileSource.java | 2 +-
.../crunch/util/CrunchRenameCopyListing.java | 4 +--
.../parquet/AvroParquetFileReaderFactoryTest.java | 2 +-
crunch-kafka/pom.xml | 2 +-
.../java/org/apache/crunch/kafka/KafkaUtils.java | 2 +-
.../scrunch/spark/ByteBufferInputStream.scala | 2 +-
crunch-test/pom.xml | 2 +-
pom.xml | 41 ++++++++++++----------
10 files changed, 38 insertions(+), 30 deletions(-)
diff --git a/crunch-core/pom.xml b/crunch-core/pom.xml
index 1f487a2..26a0707 100644
--- a/crunch-core/pom.xml
+++ b/crunch-core/pom.xml
@@ -42,7 +42,6 @@ under the License.
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro-mapred</artifactId>
- <classifier>${avro.classifier}</classifier>
</dependency>
<dependency>
@@ -69,6 +68,12 @@ under the License.
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-distcp</artifactId>
<scope>provided</scope>
</dependency>
@@ -142,7 +147,7 @@ under the License.
<dependency>
<groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
+ <artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
diff --git a/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java b/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java
index a14a5e3..1b2b151 100644
--- a/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java
+++ b/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java
@@ -153,7 +153,7 @@ public class AvroParquetFileSourceTargetIT implements Serializable {
Person person = Iterables.getOnlyElement(ageOnly.materialize());
assertNull(person.getName());
- assertEquals(person.getAge(), new Integer(42));
+ assertEquals(person.getAge(), 42);
assertNull(person.getSiblingnames());
}
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java b/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java
index 09703bd..9660b16 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java
@@ -186,7 +186,7 @@ public class AvroParquetFileSource<T extends IndexedRecord> extends FileSourceIm
if (field == null) {
throw new IllegalArgumentException("No field " + fieldName + " in schema: " + baseSchema.getName());
}
- fields.add(new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultValue(), field.order()));
+ fields.add(new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal(), field.order()));
return this;
}
diff --git a/crunch-core/src/main/java/org/apache/crunch/util/CrunchRenameCopyListing.java b/crunch-core/src/main/java/org/apache/crunch/util/CrunchRenameCopyListing.java
index b930beb..7431d5d 100644
--- a/crunch-core/src/main/java/org/apache/crunch/util/CrunchRenameCopyListing.java
+++ b/crunch-core/src/main/java/org/apache/crunch/util/CrunchRenameCopyListing.java
@@ -219,7 +219,7 @@ public class CrunchRenameCopyListing extends SimpleCopyListing {
DistCpUtils.getRelativePath(sourcePathRoot, fileStatus.getPath()), fileStatus.getPath());
}
- if (!shouldCopy(fileStatus.getPath(), options)) {
+ if (!shouldCopy(fileStatus.getPath())) {
return;
}
@@ -269,4 +269,4 @@ public class CrunchRenameCopyListing extends SimpleCopyListing {
protected long getNumberOfPaths() {
return totalPaths;
}
-}
\ No newline at end of file
+}
diff --git a/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java b/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java
index d2ad5bc..95d4561 100644
--- a/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java
+++ b/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java
@@ -98,7 +98,7 @@ public class AvroParquetFileReaderFactoryTest {
}
public static Schema.Field cloneField(Schema.Field field) {
- return new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultValue());
+ return new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal());
}
}
diff --git a/crunch-kafka/pom.xml b/crunch-kafka/pom.xml
index 14817b6..1036808 100644
--- a/crunch-kafka/pom.xml
+++ b/crunch-kafka/pom.xml
@@ -40,7 +40,7 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.11</artifactId>
+ <artifactId>kafka_${scala.base.version}</artifactId>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
diff --git a/crunch-kafka/src/main/java/org/apache/crunch/kafka/KafkaUtils.java b/crunch-kafka/src/main/java/org/apache/crunch/kafka/KafkaUtils.java
index 2681df0..0e9d750 100644
--- a/crunch-kafka/src/main/java/org/apache/crunch/kafka/KafkaUtils.java
+++ b/crunch-kafka/src/main/java/org/apache/crunch/kafka/KafkaUtils.java
@@ -34,7 +34,7 @@ import org.apache.crunch.CrunchRuntimeException;
import org.apache.hadoop.conf.Configuration;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.network.ListenerName;
-import org.apache.kafka.common.protocol.SecurityProtocol;
+import org.apache.kafka.common.security.auth.SecurityProtocol;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Option;
diff --git a/crunch-spark/src/main/scala/org/apache/crunch/scrunch/spark/ByteBufferInputStream.scala b/crunch-spark/src/main/scala/org/apache/crunch/scrunch/spark/ByteBufferInputStream.scala
index cc11b46..b43fa55 100644
--- a/crunch-spark/src/main/scala/org/apache/crunch/scrunch/spark/ByteBufferInputStream.scala
+++ b/crunch-spark/src/main/scala/org/apache/crunch/scrunch/spark/ByteBufferInputStream.scala
@@ -55,7 +55,7 @@ class ByteBufferInputStream(private var buffer: ByteBuffer)
override def skip(bytes: Long): Long = {
if (buffer != null) {
val amountToSkip = math.min(bytes, buffer.remaining).toInt
- buffer.position(buffer.position + amountToSkip)
+ buffer.position(buffer.position() + amountToSkip)
if (buffer.remaining() == 0) {
cleanUp()
}
diff --git a/crunch-test/pom.xml b/crunch-test/pom.xml
index b2c18dc..fb3110d 100644
--- a/crunch-test/pom.xml
+++ b/crunch-test/pom.xml
@@ -68,7 +68,7 @@ under the License.
<dependency>
<groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
+ <artifactId>mockito-core</artifactId>
</dependency>
<dependency>
diff --git a/pom.xml b/pom.xml
index 46d8838..05a21ad 100644
--- a/pom.xml
+++ b/pom.xml
@@ -72,8 +72,8 @@ under the License.
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <java.source.version>1.7</java.source.version>
- <java.target.version>1.7</java.target.version>
+ <java.source.version>1.8</java.source.version>
+ <java.target.version>1.8</java.target.version>
<!--
NOTE: Please don't change any dependency versions here without
checking first that they match those on Hadoop's runtime
@@ -85,7 +85,7 @@ under the License.
<commons-lang.version>2.6</commons-lang.version>
<commons-codec.version>1.4</commons-codec.version>
<commons-cli.version>1.2</commons-cli.version>
- <avro.version>1.7.7</avro.version>
+ <avro.version>1.9.1</avro.version>
<hive.version>2.1.0</hive.version>
<parquet.version>1.8.1</parquet.version>
<javassist.version>3.16.1-GA</javassist.version>
@@ -94,22 +94,21 @@ under the License.
<libthrift.version>0.8.0</libthrift.version>
<slf4j.version>1.6.1</slf4j.version>
<log4j.version>1.2.15</log4j.version>
- <algebird.version>0.9.0</algebird.version>
+ <algebird.version>0.13.2</algebird.version>
<junit.version>4.10</junit.version>
<hamcrest.version>1.1</hamcrest.version>
- <mockito.version>1.9.0</mockito.version>
+ <mockito.version>2.23.4</mockito.version>
<pkg>org.apache.crunch</pkg>
- <hadoop.version>2.7.1</hadoop.version>
+ <hadoop.version>2.8.2</hadoop.version>
<hbase.version>2.0.1</hbase.version>
- <avro.classifier>hadoop2</avro.classifier>
<hive.version>2.1.0</hive.version>
- <kafka.version>0.10.2.1</kafka.version>
- <scala.base.version>2.11</scala.base.version>
- <scala.version>2.11.8</scala.version>
- <scalatest.version>2.2.4</scalatest.version>
- <spark.version>2.0.0</spark.version>
+ <kafka.version>1.1.0</kafka.version>
+ <scala.base.version>2.12</scala.base.version>
+ <scala.version>2.12.6</scala.version>
+ <scalatest.version>3.0.1</scalatest.version>
+ <spark.version>2.4.0</spark.version>
<jline.version>2.12.1</jline.version>
<jsr305.version>1.3.9</jsr305.version>
</properties>
@@ -202,6 +201,12 @@ under the License.
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
<version>${hadoop.version}</version>
</dependency>
@@ -258,7 +263,6 @@ under the License.
<groupId>org.apache.avro</groupId>
<artifactId>avro-mapred</artifactId>
<version>${avro.version}</version>
- <classifier>${avro.classifier}</classifier>
<exclusions>
<exclusion>
<groupId>org.apache.avro</groupId>
@@ -391,7 +395,7 @@ under the License.
<dependency>
<groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
+ <artifactId>mockito-core</artifactId>
<version>${mockito.version}</version>
</dependency>
@@ -695,9 +699,9 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
- <version>2.18.1</version>
+ <version>2.22.2</version>
<configuration>
- <argLine>-Xmx2G -XX:PermSize=512m -XX:MaxPermSize=1G -Dfile.encoding=UTF-8</argLine>
+ <argLine>-Xmx2G -Dfile.encoding=UTF-8</argLine>
</configuration>
</plugin>
<plugin>
@@ -708,7 +712,7 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
- <version>2.10</version>
+ <version>2.22.2</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -837,9 +841,8 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
- <version>2.18.1</version>
+ <version>2.22.2</version>
<configuration>
- <argLine>-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m -Dfile.encoding=UTF-8</argLine>
<testSourceDirectory>${basedir}/src/it/java</testSourceDirectory>
</configuration>
<executions>