You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by cr...@apache.org on 2015/02/26 01:41:42 UTC
samza git commit: SAMZA-544;
move TestSamzaContainerPerformance to an integration test
Repository: samza
Updated Branches:
refs/heads/master 77359feb8 -> a984a6a2c
SAMZA-544; move TestSamzaContainerPerformance to an integration test
Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/a984a6a2
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/a984a6a2
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/a984a6a2
Branch: refs/heads/master
Commit: a984a6a2c1f91f0d4bccaebbd992dc4c3794a9c9
Parents: 77359fe
Author: Chris Riccomini <cr...@apache.org>
Authored: Wed Feb 25 16:41:32 2015 -0800
Committer: Chris Riccomini <cr...@apache.org>
Committed: Wed Feb 25 16:41:32 2015 -0800
----------------------------------------------------------------------
bin/integration-tests.sh | 2 +-
build.gradle | 13 ---
.../perf/container-performance.properties | 32 ++++++
samza-test/src/main/python/integration_tests.py | 30 ++++++
samza-test/src/main/python/tests.py | 30 ------
.../src/main/python/tests/performance_tests.py | 21 +++-
.../TestSamzaContainerPerformance.scala | 106 -------------------
7 files changed, 80 insertions(+), 154 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/samza/blob/a984a6a2/bin/integration-tests.sh
----------------------------------------------------------------------
diff --git a/bin/integration-tests.sh b/bin/integration-tests.sh
index 389479b..af00b5f 100755
--- a/bin/integration-tests.sh
+++ b/bin/integration-tests.sh
@@ -79,7 +79,7 @@ if [[ $SWITCHES != *"console-log-level"* ]]; then
fi
# run the tests
-zopkio --config-overrides remote_install_path=$ABS_TEST_DIR $SWITCHES $SCRIPTS_DIR/tests.py
+zopkio --config-overrides remote_install_path=$ABS_TEST_DIR $SWITCHES $SCRIPTS_DIR/integration_tests.py
# go back to execution directory
deactivate
http://git-wip-us.apache.org/repos/asf/samza/blob/a984a6a2/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index b803276..9ca6cdc 100644
--- a/build.gradle
+++ b/build.gradle
@@ -376,19 +376,6 @@ project(":samza-test_$scalaVersion") {
testRuntime "org.slf4j:slf4j-simple:$slf4jVersion"
}
- test {
- // Bump up the heap so we can start ZooKeeper and Kafka brokers. Also
- // required for TestSamzaContainerPerformance when a high thread count
- // with a lot of inputs is used.
- minHeapSize = "1560m"
- maxHeapSize = "1560m"
- jvmArgs = ["-XX:+UseConcMarkSweepGC", "-server"]
-
- // Forward all samza.* system properties to test subprocesses. This is
- // useful for configuring TestSamzaContainerPerformance from the CLI.
- systemProperties = System.properties.findAll { it.key.startsWith("samza") }
- }
-
tasks.create(name: "releaseTestJobs", dependsOn: configurations.archives.artifacts, type: Tar) {
description 'Build an integration test tarball'
compression = Compression.GZIP
http://git-wip-us.apache.org/repos/asf/samza/blob/a984a6a2/samza-test/src/main/config/perf/container-performance.properties
----------------------------------------------------------------------
diff --git a/samza-test/src/main/config/perf/container-performance.properties b/samza-test/src/main/config/perf/container-performance.properties
new file mode 100644
index 0000000..86b2d58
--- /dev/null
+++ b/samza-test/src/main/config/perf/container-performance.properties
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Job
+job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
+job.name=container-performance
+
+# YARN
+yarn.container.count=1
+yarn.container.memory.mb=4096
+
+# Task
+task.opts=-Xmx3072m -XX:+UseConcMarkSweepGC
+task.class=org.apache.samza.test.performance.TestPerformanceTask
+task.inputs=mock.stream0,mock.stream1,mock.stream2,mock.stream3,mock.stream4,mock.stream5,mock.stream6,mock.stream7,mock.stream8,mock.stream9,mock.stream10,mock.stream11,mock.stream12,mock.stream13,mock.stream14,mock.stream15,mock.stream16,mock.stream17,mock.stream18,mock.stream19,mock.stream20,mock.stream21,mock.stream22,mock.stream23,mock.stream24,mock.stream25,mock.stream26,mock.stream27,mock.stream28,mock.stream29,mock.stream30,mock.stream31,mock.stream32,mock.stream33,mock.stream34,mock.stream35,mock.stream36,mock.stream37,mock.stream38,mock.stream39,mock.stream40,mock.stream41,mock.stream42,mock.stream43,mock.stream44,mock.stream45,mock.stream46,mock.stream47,mock.stream48,mock.stream49,mock.stream50,mock.stream51,mock.stream52,mock.stream53,mock.stream54,mock.stream55,mock.stream56,mock.stream57,mock.stream58,mock.stream59,mock.stream60,mock.stream61,mock.stream62,mock.stream63,mock.stream64,mock.stream65,mock.stream66,mock.stream67,mock.stream68,mock.stream69,mock.stream70,m
ock.stream71,mock.stream72,mock.stream73,mock.stream74,mock.stream75,mock.stream76,mock.stream77,mock.stream78,mock.stream79,mock.stream80,mock.stream81,mock.stream82,mock.stream83,mock.stream84,mock.stream85,mock.stream86,mock.stream87,mock.stream88,mock.stream89,mock.stream90,mock.stream91,mock.stream92,mock.stream93,mock.stream94,mock.stream95,mock.stream96,mock.stream97,mock.stream98,mock.stream99,mock.stream100,mock.stream101,mock.stream102,mock.stream103,mock.stream104,mock.stream105,mock.stream106,mock.stream107,mock.stream108,mock.stream109,mock.stream110,mock.stream111,mock.stream112,mock.stream113,mock.stream114,mock.stream115,mock.stream116,mock.stream117,mock.stream118,mock.stream119,mock.stream120,mock.stream121,mock.stream122,mock.stream123,mock.stream124,mock.stream125,mock.stream126,mock.stream127,mock.stream128,mock.stream129,mock.stream130,mock.stream131,mock.stream132,mock.stream133,mock.stream134,mock.stream135,mock.stream136,mock.stream137,mock.stream138,mock.st
ream139,mock.stream140,mock.stream141,mock.stream142,mock.stream143,mock.stream144,mock.stream145,mock.stream146,mock.stream147,mock.stream148,mock.stream149,mock.stream150,mock.stream151,mock.stream152,mock.stream153,mock.stream154,mock.stream155,mock.stream156,mock.stream157,mock.stream158,mock.stream159,mock.stream160,mock.stream161,mock.stream162,mock.stream163,mock.stream164,mock.stream165,mock.stream166,mock.stream167,mock.stream168,mock.stream169,mock.stream170,mock.stream171,mock.stream172,mock.stream173,mock.stream174,mock.stream175,mock.stream176,mock.stream177,mock.stream178,mock.stream179,mock.stream180,mock.stream181,mock.stream182,mock.stream183,mock.stream184,mock.stream185,mock.stream186,mock.stream187,mock.stream188,mock.stream189,mock.stream190,mock.stream191,mock.stream192,mock.stream193,mock.stream194,mock.stream195,mock.stream196,mock.stream197,mock.stream198,mock.stream199,mock.stream200,mock.stream201,mock.stream202,mock.stream203,mock.stream204,mock.stream205
,mock.stream206,mock.stream207,mock.stream208,mock.stream209,mock.stream210,mock.stream211,mock.stream212,mock.stream213,mock.stream214,mock.stream215,mock.stream216,mock.stream217,mock.stream218,mock.stream219,mock.stream220,mock.stream221,mock.stream222,mock.stream223,mock.stream224,mock.stream225,mock.stream226,mock.stream227,mock.stream228,mock.stream229,mock.stream230,mock.stream231,mock.stream232,mock.stream233,mock.stream234,mock.stream235,mock.stream236,mock.stream237,mock.stream238,mock.stream239,mock.stream240,mock.stream241,mock.stream242,mock.stream243,mock.stream244,mock.stream245,mock.stream246,mock.stream247,mock.stream248,mock.stream249,mock.stream250,mock.stream251,mock.stream252,mock.stream253,mock.stream254,mock.stream255,mock.stream256,mock.stream257,mock.stream258,mock.stream259,mock.stream260,mock.stream261,mock.stream262,mock.stream263,mock.stream264,mock.stream265,mock.stream266,mock.stream267,mock.stream268,mock.stream269,mock.stream270,mock.stream271,mock.s
tream272,mock.stream273,mock.stream274,mock.stream275,mock.stream276,mock.stream277,mock.stream278,mock.stream279,mock.stream280,mock.stream281,mock.stream282,mock.stream283,mock.stream284,mock.stream285,mock.stream286,mock.stream287,mock.stream288,mock.stream289,mock.stream290,mock.stream291,mock.stream292,mock.stream293,mock.stream294,mock.stream295,mock.stream296,mock.stream297,mock.stream298,mock.stream299,mock.stream300,mock.stream301,mock.stream302,mock.stream303,mock.stream304,mock.stream305,mock.stream306,mock.stream307,mock.stream308,mock.stream309,mock.stream310,mock.stream311,mock.stream312,mock.stream313,mock.stream314,mock.stream315,mock.stream316,mock.stream317,mock.stream318,mock.stream319,mock.stream320,mock.stream321,mock.stream322,mock.stream323,mock.stream324,mock.stream325,mock.stream326,mock.stream327,mock.stream328,mock.stream329,mock.stream330,mock.stream331,mock.stream332,mock.stream333,mock.stream334,mock.stream335,mock.stream336,mock.stream337,mock.stream33
8,mock.stream339,mock.stream340,mock.stream341,mock.stream342,mock.stream343,mock.stream344,mock.stream345,mock.stream346,mock.stream347,mock.stream348,mock.stream349,mock.stream350,mock.stream351,mock.stream352,mock.stream353,mock.stream354,mock.stream355,mock.stream356,mock.stream357,mock.stream358,mock.stream359,mock.stream360,mock.stream361,mock.stream362,mock.stream363,mock.stream364,mock.stream365,mock.stream366,mock.stream367,mock.stream368,mock.stream369,mock.stream370,mock.stream371,mock.stream372,mock.stream373,mock.stream374,mock.stream375,mock.stream376,mock.stream377,mock.stream378,mock.stream379,mock.stream380,mock.stream381,mock.stream382,mock.stream383,mock.stream384,mock.stream385,mock.stream386,mock.stream387,mock.stream388,mock.stream389,mock.stream390,mock.stream391,mock.stream392,mock.stream393,mock.stream394,mock.stream395,mock.stream396,mock.stream397,mock.stream398,mock.stream399,mock.stream400,mock.stream401,mock.stream402,mock.stream403,mock.stream404,mock.
stream405,mock.stream406,mock.stream407,mock.stream408,mock.stream409,mock.stream410,mock.stream411,mock.stream412,mock.stream413,mock.stream414,mock.stream415,mock.stream416,mock.stream417,mock.stream418,mock.stream419,mock.stream420,mock.stream421,mock.stream422,mock.stream423,mock.stream424,mock.stream425,mock.stream426,mock.stream427,mock.stream428,mock.stream429,mock.stream430,mock.stream431,mock.stream432,mock.stream433,mock.stream434,mock.stream435,mock.stream436,mock.stream437,mock.stream438,mock.stream439,mock.stream440,mock.stream441,mock.stream442,mock.stream443,mock.stream444,mock.stream445,mock.stream446,mock.stream447,mock.stream448,mock.stream449,mock.stream450,mock.stream451,mock.stream452,mock.stream453,mock.stream454,mock.stream455,mock.stream456,mock.stream457,mock.stream458,mock.stream459,mock.stream460,mock.stream461,mock.stream462,mock.stream463,mock.stream464,mock.stream465,mock.stream466,mock.stream467,mock.stream468,mock.stream469,mock.stream470,mock.stream4
71,mock.stream472,mock.stream473,mock.stream474,mock.stream475,mock.stream476,mock.stream477,mock.stream478,mock.stream479,mock.stream480,mock.stream481,mock.stream482,mock.stream483,mock.stream484,mock.stream485,mock.stream486,mock.stream487,mock.stream488,mock.stream489,mock.stream490,mock.stream491,mock.stream492,mock.stream493,mock.stream494,mock.stream495,mock.stream496,mock.stream497,mock.stream498,mock.stream499,mock.stream500,mock.stream501,mock.stream502,mock.stream503,mock.stream504,mock.stream505,mock.stream506,mock.stream507,mock.stream508,mock.stream509,mock.stream510,mock.stream511,mock.stream512,mock.stream513,mock.stream514,mock.stream515,mock.stream516,mock.stream517,mock.stream518,mock.stream519,mock.stream520,mock.stream521,mock.stream522,mock.stream523,mock.stream524,mock.stream525,mock.stream526,mock.stream527,mock.stream528,mock.stream529,mock.stream530,mock.stream531,mock.stream532,mock.stream533,mock.stream534,mock.stream535,mock.stream536,mock.stream537,mock
.stream538,mock.stream539,mock.stream540,mock.stream541,mock.stream542,mock.stream543,mock.stream544,mock.stream545,mock.stream546,mock.stream547,mock.stream548,mock.stream549,mock.stream550,mock.stream551,mock.stream552,mock.stream553,mock.stream554,mock.stream555,mock.stream556,mock.stream557,mock.stream558,mock.stream559,mock.stream560,mock.stream561,mock.stream562,mock.stream563,mock.stream564,mock.stream565,mock.stream566,mock.stream567,mock.stream568,mock.stream569,mock.stream570,mock.stream571,mock.stream572,mock.stream573,mock.stream574,mock.stream575,mock.stream576,mock.stream577,mock.stream578,mock.stream579,mock.stream580,mock.stream581,mock.stream582,mock.stream583,mock.stream584,mock.stream585,mock.stream586,mock.stream587,mock.stream588,mock.stream589,mock.stream590,mock.stream591,mock.stream592,mock.stream593,mock.stream594,mock.stream595,mock.stream596,mock.stream597,mock.stream598,mock.stream599,mock.stream600,mock.stream601,mock.stream602,mock.stream603,mock.stream
604,mock.stream605,mock.stream606,mock.stream607,mock.stream608,mock.stream609,mock.stream610,mock.stream611,mock.stream612,mock.stream613,mock.stream614,mock.stream615,mock.stream616,mock.stream617,mock.stream618,mock.stream619,mock.stream620,mock.stream621,mock.stream622,mock.stream623,mock.stream624,mock.stream625,mock.stream626,mock.stream627,mock.stream628,mock.stream629,mock.stream630,mock.stream631,mock.stream632,mock.stream633,mock.stream634,mock.stream635,mock.stream636,mock.stream637,mock.stream638,mock.stream639,mock.stream640,mock.stream641,mock.stream642,mock.stream643,mock.stream644,mock.stream645,mock.stream646,mock.stream647,mock.stream648,mock.stream649,mock.stream650,mock.stream651,mock.stream652,mock.stream653,mock.stream654,mock.stream655,mock.stream656,mock.stream657,mock.stream658,mock.stream659,mock.stream660,mock.stream661,mock.stream662,mock.stream663,mock.stream664,mock.stream665,mock.stream666,mock.stream667,mock.stream668,mock.stream669,mock.stream670,moc
k.stream671,mock.stream672,mock.stream673,mock.stream674,mock.stream675,mock.stream676,mock.stream677,mock.stream678,mock.stream679,mock.stream680,mock.stream681,mock.stream682,mock.stream683,mock.stream684,mock.stream685,mock.stream686,mock.stream687,mock.stream688,mock.stream689,mock.stream690,mock.stream691,mock.stream692,mock.stream693,mock.stream694,mock.stream695,mock.stream696,mock.stream697,mock.stream698,mock.stream699,mock.stream700,mock.stream701,mock.stream702,mock.stream703,mock.stream704,mock.stream705,mock.stream706,mock.stream707,mock.stream708,mock.stream709,mock.stream710,mock.stream711,mock.stream712,mock.stream713,mock.stream714,mock.stream715,mock.stream716,mock.stream717,mock.stream718,mock.stream719,mock.stream720,mock.stream721,mock.stream722,mock.stream723,mock.stream724,mock.stream725,mock.stream726,mock.stream727,mock.stream728,mock.stream729,mock.stream730,mock.stream731,mock.stream732,mock.stream733,mock.stream734,mock.stream735,mock.stream736,mock.strea
m737,mock.stream738,mock.stream739,mock.stream740,mock.stream741,mock.stream742,mock.stream743,mock.stream744,mock.stream745,mock.stream746,mock.stream747,mock.stream748,mock.stream749,mock.stream750,mock.stream751,mock.stream752,mock.stream753,mock.stream754,mock.stream755,mock.stream756,mock.stream757,mock.stream758,mock.stream759,mock.stream760,mock.stream761,mock.stream762,mock.stream763,mock.stream764,mock.stream765,mock.stream766,mock.stream767,mock.stream768,mock.stream769,mock.stream770,mock.stream771,mock.stream772,mock.stream773,mock.stream774,mock.stream775,mock.stream776,mock.stream777,mock.stream778,mock.stream779,mock.stream780,mock.stream781,mock.stream782,mock.stream783,mock.stream784,mock.stream785,mock.stream786,mock.stream787,mock.stream788,mock.stream789,mock.stream790,mock.stream791,mock.stream792,mock.stream793,mock.stream794,mock.stream795,mock.stream796,mock.stream797,mock.stream798,mock.stream799,mock.stream800,mock.stream801,mock.stream802,mock.stream803,mo
ck.stream804,mock.stream805,mock.stream806,mock.stream807,mock.stream808,mock.stream809,mock.stream810,mock.stream811,mock.stream812,mock.stream813,mock.stream814,mock.stream815,mock.stream816,mock.stream817,mock.stream818,mock.stream819,mock.stream820,mock.stream821,mock.stream822,mock.stream823,mock.stream824,mock.stream825,mock.stream826,mock.stream827,mock.stream828,mock.stream829,mock.stream830,mock.stream831,mock.stream832,mock.stream833,mock.stream834,mock.stream835,mock.stream836,mock.stream837,mock.stream838,mock.stream839,mock.stream840,mock.stream841,mock.stream842,mock.stream843,mock.stream844,mock.stream845,mock.stream846,mock.stream847,mock.stream848,mock.stream849,mock.stream850,mock.stream851,mock.stream852,mock.stream853,mock.stream854,mock.stream855,mock.stream856,mock.stream857,mock.stream858,mock.stream859,mock.stream860,mock.stream861,mock.stream862,mock.stream863,mock.stream864,mock.stream865,mock.stream866,mock.stream867,mock.stream868,mock.stream869,mock.stre
am870,mock.stream871,mock.stream872,mock.stream873,mock.stream874,mock.stream875,mock.stream876,mock.stream877,mock.stream878,mock.stream879,mock.stream880,mock.stream881,mock.stream882,mock.stream883,mock.stream884,mock.stream885,mock.stream886,mock.stream887,mock.stream888,mock.stream889,mock.stream890,mock.stream891,mock.stream892,mock.stream893,mock.stream894,mock.stream895,mock.stream896,mock.stream897,mock.stream898,mock.stream899,mock.stream900,mock.stream901,mock.stream902,mock.stream903,mock.stream904,mock.stream905,mock.stream906,mock.stream907,mock.stream908,mock.stream909,mock.stream910,mock.stream911,mock.stream912,mock.stream913,mock.stream914,mock.stream915,mock.stream916,mock.stream917,mock.stream918,mock.stream919,mock.stream920,mock.stream921,mock.stream922,mock.stream923,mock.stream924,mock.stream925,mock.stream926,mock.stream927,mock.stream928,mock.stream929,mock.stream930,mock.stream931,mock.stream932,mock.stream933,mock.stream934,mock.stream935,mock.stream936,m
ock.stream937,mock.stream938,mock.stream939,mock.stream940,mock.stream941,mock.stream942,mock.stream943,mock.stream944,mock.stream945,mock.stream946,mock.stream947,mock.stream948,mock.stream949,mock.stream950,mock.stream951,mock.stream952,mock.stream953,mock.stream954,mock.stream955,mock.stream956,mock.stream957,mock.stream958,mock.stream959,mock.stream960,mock.stream961,mock.stream962,mock.stream963,mock.stream964,mock.stream965,mock.stream966,mock.stream967,mock.stream968,mock.stream969,mock.stream970,mock.stream971,mock.stream972,mock.stream973,mock.stream974,mock.stream975,mock.stream976,mock.stream977,mock.stream978,mock.stream979,mock.stream980,mock.stream981,mock.stream982,mock.stream983,mock.stream984,mock.stream985,mock.stream986,mock.stream987,mock.stream988,mock.stream989,mock.stream990,mock.stream991,mock.stream992,mock.stream993,mock.stream994,mock.stream995,mock.stream996,mock.stream997,mock.stream998,mock.stream999,
+
+# Kafka System
+systems.mock.samza.factory=org.apache.samza.system.mock.MockSystemFactory
http://git-wip-us.apache.org/repos/asf/samza/blob/a984a6a2/samza-test/src/main/python/integration_tests.py
----------------------------------------------------------------------
diff --git a/samza-test/src/main/python/integration_tests.py b/samza-test/src/main/python/integration_tests.py
new file mode 100644
index 0000000..df64e23
--- /dev/null
+++ b/samza-test/src/main/python/integration_tests.py
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# 'License'); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+
+dir = os.path.dirname(os.path.abspath(__file__))
+
+test = {
+ 'deployment_code': os.path.join(dir, 'deployment.py'),
+ 'perf_code': os.path.join(dir, 'perf.py'),
+ 'configs_directory': os.path.join(dir, 'configs'),
+ 'test_code': [
+ os.path.join(dir, 'tests', 'smoke_tests.py'),
+ os.path.join(dir, 'tests', 'performance_tests.py'),
+ ],
+}
http://git-wip-us.apache.org/repos/asf/samza/blob/a984a6a2/samza-test/src/main/python/tests.py
----------------------------------------------------------------------
diff --git a/samza-test/src/main/python/tests.py b/samza-test/src/main/python/tests.py
deleted file mode 100644
index df64e23..0000000
--- a/samza-test/src/main/python/tests.py
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# 'License'); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import os
-
-dir = os.path.dirname(os.path.abspath(__file__))
-
-test = {
- 'deployment_code': os.path.join(dir, 'deployment.py'),
- 'perf_code': os.path.join(dir, 'perf.py'),
- 'configs_directory': os.path.join(dir, 'configs'),
- 'test_code': [
- os.path.join(dir, 'tests', 'smoke_tests.py'),
- os.path.join(dir, 'tests', 'performance_tests.py'),
- ],
-}
http://git-wip-us.apache.org/repos/asf/samza/blob/a984a6a2/samza-test/src/main/python/tests/performance_tests.py
----------------------------------------------------------------------
diff --git a/samza-test/src/main/python/tests/performance_tests.py b/samza-test/src/main/python/tests/performance_tests.py
index a97717f..e6c873a 100644
--- a/samza-test/src/main/python/tests/performance_tests.py
+++ b/samza-test/src/main/python/tests/performance_tests.py
@@ -22,9 +22,11 @@ from kafka import SimpleProducer, SimpleConsumer
logger = logging.getLogger(__name__)
-JOB_ID = 'kafka-read-write-performance'
PACKAGE_ID = 'tests'
-CONFIG_FILE = 'config/perf/kafka-read-write-performance.properties'
+KAFKA_JOB_ID = 'kafka-read-write-performance'
+KAFKA_CONFIG_FILE = 'config/perf/kafka-read-write-performance.properties'
+CONTAINER_JOB_ID = 'container-performance'
+CONTAINER_CONFIG_FILE = 'config/perf/container-performance.properties'
TEST_INPUT_TOPIC = 'kafka-read-write-performance-input'
TEST_OUTPUT_TOPIC = 'kafka-read-write-performance-output'
NUM_MESSAGES = 1000000
@@ -36,8 +38,8 @@ def test_kafka_read_write_performance():
writes/sec for the job is logged to the job's container.
"""
_load_data()
- util.start_job(PACKAGE_ID, JOB_ID, CONFIG_FILE)
- util.await_job(PACKAGE_ID, JOB_ID)
+ util.start_job(PACKAGE_ID, KAFKA_JOB_ID, KAFKA_CONFIG_FILE)
+ util.await_job(PACKAGE_ID, KAFKA_JOB_ID)
def validate_kafka_read_write_performance():
"""
@@ -59,6 +61,17 @@ def validate_kafka_read_write_performance():
assert NUM_MESSAGES == message_count, 'Expected {0} lines, but found {1}'.format(NUM_MESSAGES, message_count)
kafka.close()
+def test_container_performance():
+ """
+ Runs TestPerformanceTask with a MockSystem to test how fast the
+ SamzaContainer can go.
+ """
+ util.start_job(PACKAGE_ID, CONTAINER_JOB_ID, CONTAINER_CONFIG_FILE)
+ util.await_job(PACKAGE_ID, CONTAINER_JOB_ID)
+
+def validate_container_performance():
+ pass
+
def _load_data():
"""
Sends 10 million messages to kafka-read-write-performance-input.
http://git-wip-us.apache.org/repos/asf/samza/blob/a984a6a2/samza-test/src/test/scala/org/apache/samza/test/performance/TestSamzaContainerPerformance.scala
----------------------------------------------------------------------
diff --git a/samza-test/src/test/scala/org/apache/samza/test/performance/TestSamzaContainerPerformance.scala b/samza-test/src/test/scala/org/apache/samza/test/performance/TestSamzaContainerPerformance.scala
deleted file mode 100644
index d589d76..0000000
--- a/samza-test/src/test/scala/org/apache/samza/test/performance/TestSamzaContainerPerformance.scala
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.samza.test.performance
-
-import org.apache.samza.config.Config
-import org.apache.samza.config.MapConfig
-import org.apache.samza.job.ShellCommandBuilder
-import org.apache.samza.job.local.ThreadJobFactory
-import org.apache.samza.task.InitableTask
-import org.apache.samza.task.MessageCollector
-import org.apache.samza.task.StreamTask
-import org.apache.samza.task.TaskCoordinator
-import org.apache.samza.task.TaskContext
-import org.apache.samza.system.IncomingMessageEnvelope
-import org.apache.samza.util.Logging
-import org.junit.Test
-
-import scala.collection.JavaConversions._
-
-/**
- * A simple unit test that drives the TestPerformanceTask. This unit test can
- * be triggered by itself using:
- *
- * <pre>
- * ./gradlew :samza-test:test -Dtest.single=TestSamzaContainerPerformance
- * <pre>
- *
- * Once the test is running, you can attach JConsole, VisualVM, or YourKit to
- * have a look at how things are behaving.
- *
- * The test can be configured with the following system properties:
- *
- * <pre>
- * samza.mock.consumer.thread.count
- * samza.mock.messages.per.batch
- * samza.mock.input.streams
- * samza.mock.partitions.per.stream
- * samza.mock.broker.sleep.ms
- * samza.task.log.interval
- * samza.task.max.messages
- * <pre>
- *
- * For example, you might specify wish to process 10000 messages simulated
- * from two input streams on one broker:
- *
- * <pre>
- * ./gradlew :samza-test:test \
- * -Dsamza.test.single=TestSamzaContainerPerformance \
- * -Psamza.mock.input.streams=2 \
- * -Psamza.mock.consumer.thread.count=1 \
- * -Psamza.task.log.interval=1000 \
- * -Psamza.task.max.messages=10000
- * <pre>
- */
-class TestSamzaContainerPerformance extends Logging{
- val consumerThreadCount = System.getProperty("samza.mock.consumer.thread.count", "12").toInt
- val messagesPerBatch = System.getProperty("samza.mock.messages.per.batch", "5000").toInt
- val streamCount = System.getProperty("samza.mock.input.streams", "1000").toInt
- val partitionsPerStreamCount = System.getProperty("samza.mock.partitions.per.stream", "4").toInt
- val brokerSleepMs = System.getProperty("samza.mock.broker.sleep.ms", "1").toInt
- var logInterval = System.getProperty("samza.task.log.interval", "10000").toInt
- var maxMessages = System.getProperty("samza.task.max.messages", "10000000").toInt
-
- val jobFactory = new ThreadJobFactory
-
- val jobConfig = Map(
- "job.factory.class" -> jobFactory.getClass.getCanonicalName,
- "job.name" -> "test-container-performance",
- "task.class" -> classOf[TestPerformanceTask].getName,
- "task.inputs" -> (0 until streamCount).map(i => "mock.stream" + i).mkString(","),
- "task.log.interval" -> logInterval.toString,
- "task.max.messages" -> maxMessages.toString,
- "systems.mock.samza.factory" -> classOf[org.apache.samza.system.mock.MockSystemFactory].getCanonicalName,
- "systems.mock.partitions.per.stream" -> partitionsPerStreamCount.toString,
- "systems.mock.messages.per.batch" -> messagesPerBatch.toString,
- "systems.mock.consumer.thread.count" -> consumerThreadCount.toString,
- "systems.mock.broker.sleep.ms" -> brokerSleepMs.toString)
-
- @Test
- def testContainerPerformance {
- info("Testing performance with configuration: %s" format jobConfig)
-
- val job = jobFactory
- .getJob(new MapConfig(jobConfig))
- .submit
-
- job.waitForFinish(Int.MaxValue)
- }
-}