You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ju...@apache.org on 2013/07/09 01:14:45 UTC
[20/43] git commit: KAFKA-844 System Test - Mirror Maker cases
enhancements; reviewed by Neha Narkhede
KAFKA-844 System Test - Mirror Maker cases enhancements; reviewed by Neha Narkhede
Project: http://git-wip-us.apache.org/repos/asf/kafka/repo
Commit: http://git-wip-us.apache.org/repos/asf/kafka/commit/afecc9f2
Tree: http://git-wip-us.apache.org/repos/asf/kafka/tree/afecc9f2
Diff: http://git-wip-us.apache.org/repos/asf/kafka/diff/afecc9f2
Branch: refs/heads/trunk
Commit: afecc9f23108b100b27017974b132331d6bab8e6
Parents: 3ead78b
Author: John Fung <fu...@gmail.com>
Authored: Thu Apr 4 16:18:50 2013 -0700
Committer: Neha Narkhede <ne...@gmail.com>
Committed: Thu Apr 4 16:19:00 2013 -0700
----------------------------------------------------------------------
.../mirror_maker_testsuite/cluster_config.json | 10 ++++++++
.../config/mirror_producer.properties | 3 ++-
.../mirror_maker_testsuite/mirror_maker_test.py | 25 ++++++++++++++------
.../testcase_5001/testcase_5001_properties.json | 22 +++++++++++++++++
.../testcase_5002/testcase_5002_properties.json | 22 +++++++++++++++++
.../testcase_5003/testcase_5003_properties.json | 13 ++++++++++
.../testcase_5004/testcase_5004_properties.json | 13 ++++++++++
.../testcase_5005/testcase_5005_properties.json | 14 +++++++++++
.../testcase_5006/testcase_5006_properties.json | 14 +++++++++++
9 files changed, 128 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kafka/blob/afecc9f2/system_test/mirror_maker_testsuite/cluster_config.json
----------------------------------------------------------------------
diff --git a/system_test/mirror_maker_testsuite/cluster_config.json b/system_test/mirror_maker_testsuite/cluster_config.json
index b679eea..5b908ff 100644
--- a/system_test/mirror_maker_testsuite/cluster_config.json
+++ b/system_test/mirror_maker_testsuite/cluster_config.json
@@ -121,6 +121,16 @@
"kafka_home": "default",
"java_home": "default",
"jmx_port": "9112"
+ },
+
+ {
+ "entity_id": "13",
+ "hostname": "localhost",
+ "role": "console_consumer",
+ "cluster_name":"source",
+ "kafka_home": "default",
+ "java_home": "default",
+ "jmx_port": "9113"
}
]
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/afecc9f2/system_test/mirror_maker_testsuite/config/mirror_producer.properties
----------------------------------------------------------------------
diff --git a/system_test/mirror_maker_testsuite/config/mirror_producer.properties b/system_test/mirror_maker_testsuite/config/mirror_producer.properties
index 7db5bfc..bdb657f 100644
--- a/system_test/mirror_maker_testsuite/config/mirror_producer.properties
+++ b/system_test/mirror_maker_testsuite/config/mirror_producer.properties
@@ -2,4 +2,5 @@ producer.type=async
queue.enqueue.timeout.ms=-1
broker.list=localhost:9094
compression.codec=0
-
+message.send.max.retries=3
+request.required.acks=1
http://git-wip-us.apache.org/repos/asf/kafka/blob/afecc9f2/system_test/mirror_maker_testsuite/mirror_maker_test.py
----------------------------------------------------------------------
diff --git a/system_test/mirror_maker_testsuite/mirror_maker_test.py b/system_test/mirror_maker_testsuite/mirror_maker_test.py
index 098f531..4dc3cdf 100644
--- a/system_test/mirror_maker_testsuite/mirror_maker_test.py
+++ b/system_test/mirror_maker_testsuite/mirror_maker_test.py
@@ -168,13 +168,13 @@ class MirrorMakerTest(ReplicationUtils, SetupUtils):
self.log_message("starting mirror makers")
kafka_system_test_utils.start_mirror_makers(self.systemTestEnv, self.testcaseEnv)
- self.anonLogger.info("sleeping for 5s")
- time.sleep(5)
+ self.anonLogger.info("sleeping for 10s")
+ time.sleep(10)
- self.log_message("creating topics")
- kafka_system_test_utils.create_topic(self.systemTestEnv, self.testcaseEnv)
- self.anonLogger.info("sleeping for 5s")
- time.sleep(5)
+ #self.log_message("creating topics")
+ #kafka_system_test_utils.create_topic(self.systemTestEnv, self.testcaseEnv)
+ #self.anonLogger.info("sleeping for 5s")
+ #time.sleep(5)
# =============================================
@@ -192,7 +192,7 @@ class MirrorMakerTest(ReplicationUtils, SetupUtils):
# =============================================
i = 1
numIterations = int(self.testcaseEnv.testcaseArgumentsDict["num_iteration"])
- bouncedEntityDownTimeSec = 1
+ bouncedEntityDownTimeSec = 15
try:
bouncedEntityDownTimeSec = int(self.testcaseEnv.testcaseArgumentsDict["bounced_entity_downtime_sec"])
except:
@@ -254,6 +254,17 @@ class MirrorMakerTest(ReplicationUtils, SetupUtils):
self.testcaseEnv.lock.release()
time.sleep(2)
+ self.anonLogger.info("sleeping for 15s")
+ time.sleep(15)
+ self.anonLogger.info("terminate Mirror Maker")
+ cmdStr = "ps auxw | grep Mirror | grep -v grep | tr -s ' ' | cut -f2 -d ' ' | xargs kill -15"
+ subproc = system_test_utils.sys_call_return_subproc(cmdStr)
+ for line in subproc.stdout.readlines():
+ line = line.rstrip('\n')
+ self.anonLogger.info("#### ["+line+"]")
+ self.anonLogger.info("sleeping for 15s")
+ time.sleep(15)
+
# =============================================
# starting consumer
# =============================================
http://git-wip-us.apache.org/repos/asf/kafka/blob/afecc9f2/system_test/mirror_maker_testsuite/testcase_5001/testcase_5001_properties.json
----------------------------------------------------------------------
diff --git a/system_test/mirror_maker_testsuite/testcase_5001/testcase_5001_properties.json b/system_test/mirror_maker_testsuite/testcase_5001/testcase_5001_properties.json
index abb500b..287cab9 100644
--- a/system_test/mirror_maker_testsuite/testcase_5001/testcase_5001_properties.json
+++ b/system_test/mirror_maker_testsuite/testcase_5001/testcase_5001_properties.json
@@ -56,6 +56,8 @@
"broker.id": "1",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_4_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_4.log",
"config_filename": "kafka_server_4.properties"
},
@@ -65,6 +67,8 @@
"broker.id": "2",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_5_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_5.log",
"config_filename": "kafka_server_5.properties"
},
@@ -74,6 +78,8 @@
"broker.id": "3",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_6_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_6.log",
"config_filename": "kafka_server_6.properties"
},
@@ -83,6 +89,8 @@
"broker.id": "4",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_7_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_7.log",
"config_filename": "kafka_server_7.properties"
},
@@ -92,6 +100,8 @@
"broker.id": "5",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_8_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_8.log",
"config_filename": "kafka_server_8.properties"
},
@@ -101,6 +111,8 @@
"broker.id": "6",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_9_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_9.log",
"config_filename": "kafka_server_9.properties"
},
@@ -114,6 +126,7 @@
"message": "500",
"request-num-acks": "-1",
"sync":"true",
+ "producer-num-retries":"5",
"log_filename": "producer_performance_10.log",
"config_filename": "producer_performance_10.properties"
},
@@ -131,6 +144,15 @@
"log_filename": "mirror_maker_12.log",
"mirror_consumer_config_filename": "mirror_consumer_12.properties",
"mirror_producer_config_filename": "mirror_producer_12.properties"
+ },
+
+ {
+ "entity_id": "13",
+ "topic": "test_1",
+ "group.id": "mytestgroup",
+ "consumer-timeout-ms": "10000",
+ "log_filename": "console_consumer_13.log",
+ "config_filename": "console_consumer_13.properties"
}
]
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/afecc9f2/system_test/mirror_maker_testsuite/testcase_5002/testcase_5002_properties.json
----------------------------------------------------------------------
diff --git a/system_test/mirror_maker_testsuite/testcase_5002/testcase_5002_properties.json b/system_test/mirror_maker_testsuite/testcase_5002/testcase_5002_properties.json
index 8d25bad..5457eb1 100644
--- a/system_test/mirror_maker_testsuite/testcase_5002/testcase_5002_properties.json
+++ b/system_test/mirror_maker_testsuite/testcase_5002/testcase_5002_properties.json
@@ -56,6 +56,8 @@
"broker.id": "1",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_4_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_4.log",
"config_filename": "kafka_server_4.properties"
},
@@ -65,6 +67,8 @@
"broker.id": "2",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_5_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_5.log",
"config_filename": "kafka_server_5.properties"
},
@@ -74,6 +78,8 @@
"broker.id": "3",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_6_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_6.log",
"config_filename": "kafka_server_6.properties"
},
@@ -83,6 +89,8 @@
"broker.id": "4",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_7_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_7.log",
"config_filename": "kafka_server_7.properties"
},
@@ -92,6 +100,8 @@
"broker.id": "5",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_8_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_8.log",
"config_filename": "kafka_server_8.properties"
},
@@ -101,6 +111,8 @@
"broker.id": "6",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_9_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_9.log",
"config_filename": "kafka_server_9.properties"
},
@@ -114,6 +126,7 @@
"message": "100",
"request-num-acks": "-1",
"sync":"true",
+ "producer-num-retries":"5",
"log_filename": "producer_performance_10.log",
"config_filename": "producer_performance_10.properties"
},
@@ -131,6 +144,15 @@
"log_filename": "mirror_maker_12.log",
"mirror_consumer_config_filename": "mirror_consumer_12.properties",
"mirror_producer_config_filename": "mirror_producer_12.properties"
+ },
+
+ {
+ "entity_id": "13",
+ "topic": "test_1",
+ "group.id": "mytestgroup",
+ "consumer-timeout-ms": "10000",
+ "log_filename": "console_consumer_13.log",
+ "config_filename": "console_consumer_13.properties"
}
]
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/afecc9f2/system_test/mirror_maker_testsuite/testcase_5003/testcase_5003_properties.json
----------------------------------------------------------------------
diff --git a/system_test/mirror_maker_testsuite/testcase_5003/testcase_5003_properties.json b/system_test/mirror_maker_testsuite/testcase_5003/testcase_5003_properties.json
index b9d72a1..98fefee 100644
--- a/system_test/mirror_maker_testsuite/testcase_5003/testcase_5003_properties.json
+++ b/system_test/mirror_maker_testsuite/testcase_5003/testcase_5003_properties.json
@@ -57,6 +57,8 @@
"broker.id": "1",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_4_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_4.log",
"config_filename": "kafka_server_4.properties"
},
@@ -66,6 +68,8 @@
"broker.id": "2",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_5_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_5.log",
"config_filename": "kafka_server_5.properties"
},
@@ -75,6 +79,8 @@
"broker.id": "3",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_6_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_6.log",
"config_filename": "kafka_server_6.properties"
},
@@ -84,6 +90,8 @@
"broker.id": "4",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_7_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_7.log",
"config_filename": "kafka_server_7.properties"
},
@@ -93,6 +101,8 @@
"broker.id": "5",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_8_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_8.log",
"config_filename": "kafka_server_8.properties"
},
@@ -102,6 +112,8 @@
"broker.id": "6",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_9_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_9.log",
"config_filename": "kafka_server_9.properties"
},
@@ -115,6 +127,7 @@
"message": "100",
"request-num-acks": "-1",
"sync":"false",
+ "producer-num-retries":"5",
"log_filename": "producer_performance_10.log",
"config_filename": "producer_performance_10.properties"
},
http://git-wip-us.apache.org/repos/asf/kafka/blob/afecc9f2/system_test/mirror_maker_testsuite/testcase_5004/testcase_5004_properties.json
----------------------------------------------------------------------
diff --git a/system_test/mirror_maker_testsuite/testcase_5004/testcase_5004_properties.json b/system_test/mirror_maker_testsuite/testcase_5004/testcase_5004_properties.json
index b72f265..6067b12 100644
--- a/system_test/mirror_maker_testsuite/testcase_5004/testcase_5004_properties.json
+++ b/system_test/mirror_maker_testsuite/testcase_5004/testcase_5004_properties.json
@@ -57,6 +57,8 @@
"broker.id": "1",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_4_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_4.log",
"config_filename": "kafka_server_4.properties"
},
@@ -66,6 +68,8 @@
"broker.id": "2",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_5_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_5.log",
"config_filename": "kafka_server_5.properties"
},
@@ -75,6 +79,8 @@
"broker.id": "3",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_6_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_6.log",
"config_filename": "kafka_server_6.properties"
},
@@ -84,6 +90,8 @@
"broker.id": "4",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_7_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_7.log",
"config_filename": "kafka_server_7.properties"
},
@@ -93,6 +101,8 @@
"broker.id": "5",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_8_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_8.log",
"config_filename": "kafka_server_8.properties"
},
@@ -102,6 +112,8 @@
"broker.id": "6",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_9_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_9.log",
"config_filename": "kafka_server_9.properties"
},
@@ -115,6 +127,7 @@
"message": "100",
"request-num-acks": "1",
"sync":"false",
+ "producer-num-retries":"5",
"log_filename": "producer_performance_10.log",
"config_filename": "producer_performance_10.properties"
},
http://git-wip-us.apache.org/repos/asf/kafka/blob/afecc9f2/system_test/mirror_maker_testsuite/testcase_5005/testcase_5005_properties.json
----------------------------------------------------------------------
diff --git a/system_test/mirror_maker_testsuite/testcase_5005/testcase_5005_properties.json b/system_test/mirror_maker_testsuite/testcase_5005/testcase_5005_properties.json
index 83ca969..58483ad 100644
--- a/system_test/mirror_maker_testsuite/testcase_5005/testcase_5005_properties.json
+++ b/system_test/mirror_maker_testsuite/testcase_5005/testcase_5005_properties.json
@@ -57,6 +57,8 @@
"broker.id": "1",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_4_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_4.log",
"config_filename": "kafka_server_4.properties"
},
@@ -66,6 +68,8 @@
"broker.id": "2",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_5_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_5.log",
"config_filename": "kafka_server_5.properties"
},
@@ -75,6 +79,8 @@
"broker.id": "3",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_6_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_6.log",
"config_filename": "kafka_server_6.properties"
},
@@ -84,6 +90,8 @@
"broker.id": "4",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_7_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_7.log",
"config_filename": "kafka_server_7.properties"
},
@@ -93,6 +101,8 @@
"broker.id": "5",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_8_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_8.log",
"config_filename": "kafka_server_8.properties"
},
@@ -102,6 +112,8 @@
"broker.id": "6",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_9_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_9.log",
"config_filename": "kafka_server_9.properties"
},
@@ -115,6 +127,7 @@
"message": "100",
"request-num-acks": "-1",
"sync":"false",
+ "producer-num-retries":"5",
"log_filename": "producer_performance_10.log",
"config_filename": "producer_performance_10.properties"
},
@@ -127,6 +140,7 @@
"message": "100",
"request-num-acks": "-1",
"sync":"false",
+ "producer-num-retries":"5",
"log_filename": "producer_performance_11.log",
"config_filename": "producer_performance_11.properties"
},
http://git-wip-us.apache.org/repos/asf/kafka/blob/afecc9f2/system_test/mirror_maker_testsuite/testcase_5006/testcase_5006_properties.json
----------------------------------------------------------------------
diff --git a/system_test/mirror_maker_testsuite/testcase_5006/testcase_5006_properties.json b/system_test/mirror_maker_testsuite/testcase_5006/testcase_5006_properties.json
index fe32712..1d9190c 100644
--- a/system_test/mirror_maker_testsuite/testcase_5006/testcase_5006_properties.json
+++ b/system_test/mirror_maker_testsuite/testcase_5006/testcase_5006_properties.json
@@ -57,6 +57,8 @@
"broker.id": "1",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_4_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_4.log",
"config_filename": "kafka_server_4.properties"
},
@@ -66,6 +68,8 @@
"broker.id": "2",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_5_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_5.log",
"config_filename": "kafka_server_5.properties"
},
@@ -75,6 +79,8 @@
"broker.id": "3",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_6_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_6.log",
"config_filename": "kafka_server_6.properties"
},
@@ -84,6 +90,8 @@
"broker.id": "4",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_7_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_7.log",
"config_filename": "kafka_server_7.properties"
},
@@ -93,6 +101,8 @@
"broker.id": "5",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_8_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_8.log",
"config_filename": "kafka_server_8.properties"
},
@@ -102,6 +112,8 @@
"broker.id": "6",
"log.segment.bytes": "10240",
"log.dir": "/tmp/kafka_server_9_logs",
+ "default.replication.factor": "3",
+ "num.partitions": "5",
"log_filename": "kafka_server_9.log",
"config_filename": "kafka_server_9.properties"
},
@@ -115,6 +127,7 @@
"message": "100",
"request-num-acks": "1",
"sync":"false",
+ "producer-num-retries":"5",
"log_filename": "producer_performance_10.log",
"config_filename": "producer_performance_10.properties"
},
@@ -127,6 +140,7 @@
"message": "100",
"request-num-acks": "1",
"sync":"false",
+ "producer-num-retries":"5",
"log_filename": "producer_performance_11.log",
"config_filename": "producer_performance_11.properties"
},