You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by jw...@apache.org on 2019/11/15 00:48:55 UTC
[crunch] branch jwills_great_version_upgrade updated (65afa91 ->
362853d)
This is an automated email from the ASF dual-hosted git repository.
jwills pushed a change to branch jwills_great_version_upgrade
in repository https://gitbox.apache.org/repos/asf/crunch.git.
from 65afa91 and more more fixes
new cd32244 mostly kafka fixes; some jackson fixes
new f49fa8e oops should have fixed that one
new 362853d Fix unnecessary stubbings in the kafka test suite
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
crunch-core/pom.xml | 5 +++++
.../test/java/org/apache/crunch/kafka/KafkaRecordsIterableIT.java | 2 --
.../org/apache/crunch/kafka/record/KafkaRecordReaderTest.java | 5 +----
.../org/apache/crunch/kafka/record/KafkaRecordsIterableIT.java | 4 +---
.../org/apache/crunch/kafka/utils/KafkaBrokerTestHarness.java | 1 +
pom.xml | 8 +++++++-
6 files changed, 15 insertions(+), 10 deletions(-)
[crunch] 03/03: Fix unnecessary stubbings in the kafka test suite
Posted by jw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jwills pushed a commit to branch jwills_great_version_upgrade
in repository https://gitbox.apache.org/repos/asf/crunch.git
commit 362853da42b73d7eae5307269be60ca8ae814a02
Author: Josh Wills <jw...@apache.org>
AuthorDate: Thu Nov 14 16:48:33 2019 -0800
Fix unnecessary stubbings in the kafka test suite
---
.../test/java/org/apache/crunch/kafka/KafkaRecordsIterableIT.java | 2 --
.../java/org/apache/crunch/kafka/record/KafkaRecordReaderTest.java | 5 +----
2 files changed, 1 insertion(+), 6 deletions(-)
diff --git a/crunch-kafka/src/test/java/org/apache/crunch/kafka/KafkaRecordsIterableIT.java b/crunch-kafka/src/test/java/org/apache/crunch/kafka/KafkaRecordsIterableIT.java
index fd940f3..dd179ae 100644
--- a/crunch-kafka/src/test/java/org/apache/crunch/kafka/KafkaRecordsIterableIT.java
+++ b/crunch-kafka/src/test/java/org/apache/crunch/kafka/KafkaRecordsIterableIT.java
@@ -385,8 +385,6 @@ public class KafkaRecordsIterableIT {
entry.getKey().partition(), entry.getValue() + 1, "key", null));
}
- when(records.isEmpty()).thenReturn(false);
- when(records.iterator()).thenReturn(returnedRecords.iterator());
when(mockedConsumer.poll(Matchers.anyLong()))
//for the fill poll call
.thenReturn(null)
diff --git a/crunch-kafka/src/test/java/org/apache/crunch/kafka/record/KafkaRecordReaderTest.java b/crunch-kafka/src/test/java/org/apache/crunch/kafka/record/KafkaRecordReaderTest.java
index fd8cd8e..4b3a548 100644
--- a/crunch-kafka/src/test/java/org/apache/crunch/kafka/record/KafkaRecordReaderTest.java
+++ b/crunch-kafka/src/test/java/org/apache/crunch/kafka/record/KafkaRecordReaderTest.java
@@ -63,9 +63,6 @@ public class KafkaRecordReaderTest {
inputSplit = new KafkaInputSplit(topicPartition.topic(), topicPartition.partition(), startOffset, endOffset);
- when(consumer.beginningOffsets(Collections.singleton(inputSplit.getTopicPartition()))).thenReturn(
- Collections.singletonMap(inputSplit.getTopicPartition(), 0L));
-
records = new ConsumerRecords<>(Collections.singletonMap(inputSplit.getTopicPartition(),
Collections.singletonList(new ConsumerRecord<>("topic", 0, 0, "key", "value"))));
@@ -166,4 +163,4 @@ public class KafkaRecordReaderTest {
return consumer;
}
}
-}
\ No newline at end of file
+}
[crunch] 02/03: oops should have fixed that one
Posted by jw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jwills pushed a commit to branch jwills_great_version_upgrade
in repository https://gitbox.apache.org/repos/asf/crunch.git
commit f49fa8eafeeb5614e0469ececadb727f858f18fc
Author: Josh Wills <jw...@apache.org>
AuthorDate: Thu Nov 14 16:34:52 2019 -0800
oops should have fixed that one
---
.../java/org/apache/crunch/kafka/record/KafkaRecordsIterableIT.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/crunch-kafka/src/test/java/org/apache/crunch/kafka/record/KafkaRecordsIterableIT.java b/crunch-kafka/src/test/java/org/apache/crunch/kafka/record/KafkaRecordsIterableIT.java
index 9939d64..f996d96 100644
--- a/crunch-kafka/src/test/java/org/apache/crunch/kafka/record/KafkaRecordsIterableIT.java
+++ b/crunch-kafka/src/test/java/org/apache/crunch/kafka/record/KafkaRecordsIterableIT.java
@@ -388,8 +388,6 @@ public class KafkaRecordsIterableIT {
entry.getKey().partition(), entry.getValue() + 1, "key", null));
}
- when(records.isEmpty()).thenReturn(false);
- when(records.iterator()).thenReturn(returnedRecords.iterator());
when(mockedConsumer.poll(Matchers.anyLong()))
//for the fill poll call
.thenReturn(null)
@@ -413,4 +411,4 @@ public class KafkaRecordsIterableIT {
private static Map<TopicPartition, Long> getStartOffsets(Properties props, String topic) {
return KafkaUtils.getBrokerOffsets(props, OffsetRequest.EarliestTime(), topic);
}
-}
\ No newline at end of file
+}
[crunch] 01/03: mostly kafka fixes; some jackson fixes
Posted by jw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jwills pushed a commit to branch jwills_great_version_upgrade
in repository https://gitbox.apache.org/repos/asf/crunch.git
commit cd32244ec96a0f1ca21ebc765c4a45d0dee6e5cf
Author: Josh Wills <jw...@apache.org>
AuthorDate: Thu Nov 14 16:22:49 2019 -0800
mostly kafka fixes; some jackson fixes
---
crunch-core/pom.xml | 5 +++++
.../org/apache/crunch/kafka/utils/KafkaBrokerTestHarness.java | 1 +
pom.xml | 8 +++++++-
3 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/crunch-core/pom.xml b/crunch-core/pom.xml
index 26a0707..c356dce 100644
--- a/crunch-core/pom.xml
+++ b/crunch-core/pom.xml
@@ -50,6 +50,11 @@ under the License.
</dependency>
<dependency>
+ <groupId>com.thoughtworks.paranamer</groupId>
+ <artifactId>paranamer</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>org.javassist</groupId>
<artifactId>javassist</artifactId>
</dependency>
diff --git a/crunch-kafka/src/test/java/org/apache/crunch/kafka/utils/KafkaBrokerTestHarness.java b/crunch-kafka/src/test/java/org/apache/crunch/kafka/utils/KafkaBrokerTestHarness.java
index 13a4e2c..f3f2234 100644
--- a/crunch-kafka/src/test/java/org/apache/crunch/kafka/utils/KafkaBrokerTestHarness.java
+++ b/crunch-kafka/src/test/java/org/apache/crunch/kafka/utils/KafkaBrokerTestHarness.java
@@ -323,6 +323,7 @@ public class KafkaBrokerTestHarness extends ZookeeperTestHarness {
props.setProperty("num.partitions", String.valueOf(PARTITIONS_PER_TOPIC));
props.setProperty("default.replication.factor", String.valueOf(brokers));
props.setProperty("auto.create.topics.enable", Boolean.FALSE.toString());
+ props.setProperty("offsets.topic.replication.factor", String.valueOf(brokers));
props.putAll(baseProperties);
diff --git a/pom.xml b/pom.xml
index 9b8b526..447f756 100644
--- a/pom.xml
+++ b/pom.xml
@@ -92,7 +92,7 @@ under the License.
<jackson.databind.version>2.6.1</jackson.databind.version>
<protobuf-java.version>2.5.0</protobuf-java.version>
<libthrift.version>0.8.0</libthrift.version>
- <slf4j.version>1.6.1</slf4j.version>
+ <slf4j.version>1.7.25</slf4j.version>
<log4j.version>1.2.15</log4j.version>
<algebird.version>0.13.2</algebird.version>
<junit.version>4.10</junit.version>
@@ -272,6 +272,12 @@ under the License.
</dependency>
<dependency>
+ <groupId>com.thoughtworks.paranamer</groupId>
+ <artifactId>paranamer</artifactId>
+ <version>2.8</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-avro</artifactId>
<version>${parquet.version}</version>