You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2018/03/14 03:34:08 UTC
atlas git commit: ATLAS-2495: updated Kafka version to 1.0.0
Repository: atlas
Updated Branches:
refs/heads/master ac2c1a779 -> 60789da3b
ATLAS-2495: updated Kafka version to 1.0.0
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/60789da3
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/60789da3
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/60789da3
Branch: refs/heads/master
Commit: 60789da3b95366a256ae1020241bb9a9ec0817de
Parents: ac2c1a7
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Sun Mar 11 17:34:08 2018 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Tue Mar 13 17:40:40 2018 -0700
----------------------------------------------------------------------
.../src/test/resources/atlas-application.properties | 1 +
.../src/test/resources/atlas-application.properties | 1 +
.../src/test/resources/atlas-application.properties | 1 +
.../src/test/resources/atlas-application.properties | 1 +
.../src/test/resources/atlas-application.properties | 1 +
.../src/test/resources/atlas-application.properties | 1 +
.../src/test/resources/atlas-application.properties | 1 +
.../src/test/resources/atlas-application.properties | 1 +
intg/src/test/resources/atlas-application.properties | 1 +
.../org/apache/atlas/kafka/EmbeddedKafkaServer.java | 15 +++++++++++++--
.../atlas/kafka/KafkaNotificationMockTest.java | 3 ++-
pom.xml | 2 +-
.../apache/atlas/web/integration/BaseResourceIT.java | 2 --
.../src/test/resources/atlas-application.properties | 1 +
14 files changed, 26 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/addons/falcon-bridge/src/test/resources/atlas-application.properties
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/test/resources/atlas-application.properties b/addons/falcon-bridge/src/test/resources/atlas-application.properties
index d1c3f71..d9fd0cc 100644
--- a/addons/falcon-bridge/src/test/resources/atlas-application.properties
+++ b/addons/falcon-bridge/src/test/resources/atlas-application.properties
@@ -79,6 +79,7 @@ atlas.kafka.entities.group.id=atlas_entities
atlas.kafka.enable.auto.commit=false
atlas.kafka.auto.offset.reset=earliest
atlas.kafka.session.timeout.ms=30000
+atlas.kafka.offsets.topic.replication.factor=1
http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/addons/hbase-bridge/src/test/resources/atlas-application.properties
----------------------------------------------------------------------
diff --git a/addons/hbase-bridge/src/test/resources/atlas-application.properties b/addons/hbase-bridge/src/test/resources/atlas-application.properties
index d1c3f71..d9fd0cc 100644
--- a/addons/hbase-bridge/src/test/resources/atlas-application.properties
+++ b/addons/hbase-bridge/src/test/resources/atlas-application.properties
@@ -79,6 +79,7 @@ atlas.kafka.entities.group.id=atlas_entities
atlas.kafka.enable.auto.commit=false
atlas.kafka.auto.offset.reset=earliest
atlas.kafka.session.timeout.ms=30000
+atlas.kafka.offsets.topic.replication.factor=1
http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/addons/hive-bridge/src/test/resources/atlas-application.properties
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/resources/atlas-application.properties b/addons/hive-bridge/src/test/resources/atlas-application.properties
index d1c3f71..d9fd0cc 100644
--- a/addons/hive-bridge/src/test/resources/atlas-application.properties
+++ b/addons/hive-bridge/src/test/resources/atlas-application.properties
@@ -79,6 +79,7 @@ atlas.kafka.entities.group.id=atlas_entities
atlas.kafka.enable.auto.commit=false
atlas.kafka.auto.offset.reset=earliest
atlas.kafka.session.timeout.ms=30000
+atlas.kafka.offsets.topic.replication.factor=1
http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/addons/sqoop-bridge/src/test/resources/atlas-application.properties
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/src/test/resources/atlas-application.properties b/addons/sqoop-bridge/src/test/resources/atlas-application.properties
index d1c3f71..d9fd0cc 100644
--- a/addons/sqoop-bridge/src/test/resources/atlas-application.properties
+++ b/addons/sqoop-bridge/src/test/resources/atlas-application.properties
@@ -79,6 +79,7 @@ atlas.kafka.entities.group.id=atlas_entities
atlas.kafka.enable.auto.commit=false
atlas.kafka.auto.offset.reset=earliest
atlas.kafka.session.timeout.ms=30000
+atlas.kafka.offsets.topic.replication.factor=1
http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/addons/storm-bridge/src/test/resources/atlas-application.properties
----------------------------------------------------------------------
diff --git a/addons/storm-bridge/src/test/resources/atlas-application.properties b/addons/storm-bridge/src/test/resources/atlas-application.properties
index 6e9042a..b92c791 100644
--- a/addons/storm-bridge/src/test/resources/atlas-application.properties
+++ b/addons/storm-bridge/src/test/resources/atlas-application.properties
@@ -81,6 +81,7 @@ atlas.kafka.entities.group.id=atlas_entities
atlas.kafka.enable.auto.commit=false
atlas.kafka.auto.offset.reset=earliest
atlas.kafka.session.timeout.ms=30000
+atlas.kafka.offsets.topic.replication.factor=1
http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/authorization/src/test/resources/atlas-application.properties
----------------------------------------------------------------------
diff --git a/authorization/src/test/resources/atlas-application.properties b/authorization/src/test/resources/atlas-application.properties
index e1357b1..3e41581 100644
--- a/authorization/src/test/resources/atlas-application.properties
+++ b/authorization/src/test/resources/atlas-application.properties
@@ -97,6 +97,7 @@ atlas.kafka.entities.group.id=atlas_entities
atlas.kafka.enable.auto.commit=false
atlas.kafka.auto.offset.reset=earliest
atlas.kafka.session.timeout.ms=30000
+atlas.kafka.offsets.topic.replication.factor=1
http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/graphdb/janus/src/test/resources/atlas-application.properties
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/atlas-application.properties b/graphdb/janus/src/test/resources/atlas-application.properties
index 5c9bd7a..a355234 100644
--- a/graphdb/janus/src/test/resources/atlas-application.properties
+++ b/graphdb/janus/src/test/resources/atlas-application.properties
@@ -74,6 +74,7 @@ atlas.kafka.consumer.timeout.ms=100
atlas.kafka.auto.commit.interval.ms=100
atlas.kafka.hook.group.id=atlas
atlas.kafka.entities.group.id=atlas_entities
+atlas.kafka.offsets.topic.replication.factor=1
######### Entity Audit Configs #########
atlas.audit.hbase.tablename=ATLAS_ENTITY_AUDIT_EVENTS
http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/graphdb/titan0/src/test/resources/atlas-application.properties
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/test/resources/atlas-application.properties b/graphdb/titan0/src/test/resources/atlas-application.properties
index c6ea4e0..6188873 100644
--- a/graphdb/titan0/src/test/resources/atlas-application.properties
+++ b/graphdb/titan0/src/test/resources/atlas-application.properties
@@ -75,6 +75,7 @@ atlas.kafka.consumer.timeout.ms=100
atlas.kafka.auto.commit.interval.ms=100
atlas.kafka.hook.group.id=atlas
atlas.kafka.entities.group.id=atlas_entities
+atlas.kafka.offsets.topic.replication.factor=1
######### Entity Audit Configs #########
atlas.audit.hbase.tablename=ATLAS_ENTITY_AUDIT_EVENTS
http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/intg/src/test/resources/atlas-application.properties
----------------------------------------------------------------------
diff --git a/intg/src/test/resources/atlas-application.properties b/intg/src/test/resources/atlas-application.properties
index 9015f89..5470163 100644
--- a/intg/src/test/resources/atlas-application.properties
+++ b/intg/src/test/resources/atlas-application.properties
@@ -97,6 +97,7 @@ atlas.kafka.entities.group.id=atlas_entities
atlas.kafka.enable.auto.commit=false
atlas.kafka.auto.offset.reset=earliest
atlas.kafka.session.timeout.ms=30000
+atlas.kafka.offsets.topic.replication.factor=1
http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/notification/src/main/java/org/apache/atlas/kafka/EmbeddedKafkaServer.java
----------------------------------------------------------------------
diff --git a/notification/src/main/java/org/apache/atlas/kafka/EmbeddedKafkaServer.java b/notification/src/main/java/org/apache/atlas/kafka/EmbeddedKafkaServer.java
index 33c8296..32b597f 100644
--- a/notification/src/main/java/org/apache/atlas/kafka/EmbeddedKafkaServer.java
+++ b/notification/src/main/java/org/apache/atlas/kafka/EmbeddedKafkaServer.java
@@ -17,15 +17,16 @@
*/
package org.apache.atlas.kafka;
+import kafka.metrics.KafkaMetricsReporter;
import kafka.server.KafkaConfig;
import kafka.server.KafkaServer;
-import kafka.utils.Time;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.service.Service;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationConverter;
import org.apache.kafka.clients.producer.*;
+import org.apache.kafka.common.utils.Time;
import org.apache.zookeeper.server.NIOServerCnxnFactory;
import org.apache.zookeeper.server.ServerCnxnFactory;
import org.apache.zookeeper.server.ZooKeeperServer;
@@ -34,6 +35,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import scala.Option;
+import scala.collection.mutable.Buffer;
import javax.inject.Inject;
import java.io.File;
@@ -43,6 +45,7 @@ import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.*;
+import java.util.concurrent.TimeUnit;
@Component
@@ -135,7 +138,10 @@ public class EmbeddedKafkaServer implements Service {
brokerConfig.setProperty("log.dirs", constructDir("kafka").getAbsolutePath());
brokerConfig.setProperty("log.flush.interval.messages", String.valueOf(1));
- kafkaServer = new KafkaServer(KafkaConfig.fromProps(brokerConfig), new SystemTime(), Option.apply(this.getClass().getName()));
+ List<KafkaMetricsReporter> metrics = new ArrayList<>();
+ Buffer<KafkaMetricsReporter> metricsReporters = scala.collection.JavaConversions.asScalaBuffer(metrics);
+
+ kafkaServer = new KafkaServer(KafkaConfig.fromProps(brokerConfig), new SystemTime(), Option.apply(this.getClass().getName()), metricsReporters);
kafkaServer.startup();
@@ -174,6 +180,11 @@ public class EmbeddedKafkaServer implements Service {
}
@Override
+ public long hiResClockMs() {
+ return TimeUnit.NANOSECONDS.toMillis(nanoseconds());
+ }
+
+ @Override
public void sleep(long arg0) {
try {
Thread.sleep(arg0);
http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationMockTest.java
----------------------------------------------------------------------
diff --git a/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationMockTest.java b/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationMockTest.java
index 78d2a90..263903b 100644
--- a/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationMockTest.java
+++ b/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationMockTest.java
@@ -77,7 +77,8 @@ public class KafkaNotificationMockTest {
String topicName = kafkaNotification.getTopicName(NotificationInterface.NotificationType.HOOK);
String message = "This is a test message";
Future returnValue = mock(Future.class);
- when(returnValue.get()).thenReturn(new RecordMetadata(new TopicPartition(topicName, 0), 0, 0));
+ TopicPartition topicPartition = new TopicPartition(topicName, 0);
+ when(returnValue.get()).thenReturn(new RecordMetadata(topicPartition, 0, 0, 0, Long.valueOf(0), 0, 0));
ProducerRecord expectedRecord = new ProducerRecord(topicName, message);
when(producer.send(expectedRecord)).thenReturn(returnValue);
http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 34e1ad0..9ddc8e1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -546,7 +546,7 @@
<hadoop.version>2.7.1</hadoop.version>
<hbase.version>1.1.2</hbase.version>
<solr.version>5.5.1</solr.version>
- <kafka.version>0.10.0.0</kafka.version>
+ <kafka.version>1.0.0</kafka.version>
<kafka.scala.binary.version>2.11</kafka.scala.binary.version>
<curator.version>2.11.0</curator.version>
<zookeeper.version>3.4.6</zookeeper.version>
http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java
index 6a7aece..6f6a7ee 100755
--- a/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java
@@ -36,7 +36,6 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
import org.apache.atlas.notification.NotificationInterface;
-import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.v1.model.instance.Id;
import org.apache.atlas.v1.model.instance.Referenceable;
import org.apache.atlas.v1.model.instance.Struct;
@@ -57,7 +56,6 @@ import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
-import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/webapp/src/test/resources/atlas-application.properties
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/atlas-application.properties b/webapp/src/test/resources/atlas-application.properties
index 402d362..1e251bc 100644
--- a/webapp/src/test/resources/atlas-application.properties
+++ b/webapp/src/test/resources/atlas-application.properties
@@ -80,6 +80,7 @@ atlas.kafka.entities.group.id=atlas_entities
atlas.kafka.enable.auto.commit=false
atlas.kafka.auto.offset.reset=earliest
atlas.kafka.session.timeout.ms=30000
+atlas.kafka.offsets.topic.replication.factor=1