You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by sa...@apache.org on 2021/11/11 21:42:51 UTC
[samza] branch master updated: SAMZA-2708: Update yarn from 2.7.1
to 2.10.1 (#1557)
This is an automated email from the ASF dual-hosted git repository.
saniljain15 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/samza.git
The following commit(s) were added to refs/heads/master by this push:
new a6ebddd SAMZA-2708: Update yarn from 2.7.1 to 2.10.1 (#1557)
a6ebddd is described below
commit a6ebdddc74fcf89c1654c8f81126e92502a5d059
Author: Tony Brusseau <93...@users.noreply.github.com>
AuthorDate: Thu Nov 11 13:42:42 2021 -0800
SAMZA-2708: Update yarn from 2.7.1 to 2.10.1 (#1557)
The version of YARN used by the Samza project is outdated and needs to be updated.
Co-authored-by: Tony Brusseau <tb...@tbrussea-mn1.linkedin.biz>
---
build.gradle | 2 +-
gradle/dependency-versions.gradle | 2 +-
samza-test/src/main/config/join/README | 8 ++++----
samza-test/src/main/python/configs/tests.json | 2 +-
samza-test/src/main/python/stream_processor.py | 2 +-
.../org/apache/samza/job/yarn/util/MockNMClient.java | 18 ++++++++++++++++++
.../job/yarn/TestSamzaYarnAppMasterLifecycle.scala | 4 +++-
7 files changed, 29 insertions(+), 9 deletions(-)
diff --git a/build.gradle b/build.gradle
index 2c4b67d..2ed9b92 100644
--- a/build.gradle
+++ b/build.gradle
@@ -509,7 +509,7 @@ project(":samza-yarn_$scalaSuffix") {
repositories {
maven {
- url "http://repo.typesafe.com/typesafe/releases"
+ url "https://repo.typesafe.com/typesafe/releases"
}
}
diff --git a/gradle/dependency-versions.gradle b/gradle/dependency-versions.gradle
index 019cde3..e18fc7d 100644
--- a/gradle/dependency-versions.gradle
+++ b/gradle/dependency-versions.gradle
@@ -45,7 +45,7 @@
rocksdbVersion = "6.6.4"
scalaTestVersion = "3.0.1"
slf4jVersion = "1.7.7"
- yarnVersion = "2.7.1"
+ yarnVersion = "2.10.1"
zkClientVersion = "0.11"
zookeeperVersion = "3.4.13"
failsafeVersion = "2.4.0"
diff --git a/samza-test/src/main/config/join/README b/samza-test/src/main/config/join/README
index 1bb220e..110587f 100644
--- a/samza-test/src/main/config/join/README
+++ b/samza-test/src/main/config/join/README
@@ -44,17 +44,17 @@ Deploy Zookeeper, YARN and Kafka:
> cd $HELLO_SAMZA_SRC
> for i in zookeeper kafka yarn; do ./bin/grid install $i; ./bin/grid start $i; done
-Update the "yarn.package.path" to $DEPLOY_DIR/samza-test_2.11-1.6.0-SNAPSHOT.tgz
+Update the "yarn.package.path" to $DEPLOY_DIR/samza-test_2.11-1.7.0-SNAPSHOT.tgz
> cd $SAMZA_SRC
> vi samza-test/src/main/config/join/common.properties
-yarn.package.path=file:///path/to/samza-hello-samza/deploy/samza-test_2.11-1.6.0-SNAPSHOT.tgz
+yarn.package.path=file:///path/to/samza-hello-samza/deploy/samza-test_2.11-1.7.0-SNAPSHOT.tgz
Then release and extract the test tarball:
> cd $SAMZA_SRC
> ./gradlew releaseTestJobs
-> cp samza-test/build/distributions/samza-test_2.11-1.6.0-SNAPSHOT.tgz $DEPLOY_DIR
+> cp samza-test/build/distributions/samza-test_2.11-1.7.0-SNAPSHOT.tgz $DEPLOY_DIR
> mkdir $DEPLOY_DIR/samza
-> tar -xvf $DEPLOY_DIR/samza-test_2.11-1.6.0-SNAPSHOT.tgz -C $DEPLOY_DIR/samza
+> tar -xvf $DEPLOY_DIR/samza-test_2.11-1.7.0-SNAPSHOT.tgz -C $DEPLOY_DIR/samza
Finally, create the kafka topics and start the samza jobs:
> ./bin/setup-int-test.sh $DEPLOY_DIR
diff --git a/samza-test/src/main/python/configs/tests.json b/samza-test/src/main/python/configs/tests.json
index 7f5d5da..cdf4861 100644
--- a/samza-test/src/main/python/configs/tests.json
+++ b/samza-test/src/main/python/configs/tests.json
@@ -1,5 +1,5 @@
{
- "samza_executable": "samza-test_2.11-1.6.0-SNAPSHOT.tgz",
+ "samza_executable": "samza-test_2.11-1.7.0-SNAPSHOT.tgz",
"samza_install_path": "deploy/smoke_tests",
"samza_config_loader_factory": "org.apache.samza.config.loaders.PropertiesConfigLoaderFactory"
}
diff --git a/samza-test/src/main/python/stream_processor.py b/samza-test/src/main/python/stream_processor.py
index f42590b..34296ef 100644
--- a/samza-test/src/main/python/stream_processor.py
+++ b/samza-test/src/main/python/stream_processor.py
@@ -43,7 +43,7 @@ class StreamProcessor:
logger.info('Running processor start command: {0}'.format(self.processor_start_command))
self.deployment_config = {
'install_path': os.path.join(runtime.get_active_config('remote_install_path'), 'deploy/{0}'.format(self.processor_id)),
- 'executable': 'samza-test_2.11-1.6.0-SNAPSHOT.tgz',
+ 'executable': 'samza-test_2.11-1.7.0-SNAPSHOT.tgz',
'post_install_cmds': [],
'start_command': self.processor_start_command,
'stop_command': '',
diff --git a/samza-yarn/src/test/java/org/apache/samza/job/yarn/util/MockNMClient.java b/samza-yarn/src/test/java/org/apache/samza/job/yarn/util/MockNMClient.java
index 32735f0..642d709 100644
--- a/samza-yarn/src/test/java/org/apache/samza/job/yarn/util/MockNMClient.java
+++ b/samza-yarn/src/test/java/org/apache/samza/job/yarn/util/MockNMClient.java
@@ -47,4 +47,22 @@ public class MockNMClient extends NMClient {
@Override
public void cleanupRunningContainersOnStop(boolean enabled) { }
+
+ @Override
+ public void commitLastReInitialization(ContainerId containerId) { }
+
+ @Override
+ public void rollbackLastReInitialization(ContainerId containerId) { }
+
+ @Override
+ public void restartContainer(ContainerId containerId) { }
+
+ @Override
+ public void increaseContainerResource(Container container) { }
+
+ @Override
+ public void updateContainerResource(Container container) { }
+
+ @Override
+ public void reInitializeContainer(ContainerId containerId, ContainerLaunchContext containerLaunchContext, boolean shouldAutoCommit) { }
}
diff --git a/samza-yarn/src/test/scala/org/apache/samza/job/yarn/TestSamzaYarnAppMasterLifecycle.scala b/samza-yarn/src/test/scala/org/apache/samza/job/yarn/TestSamzaYarnAppMasterLifecycle.scala
index 5f78f78..67e018a 100644
--- a/samza-yarn/src/test/scala/org/apache/samza/job/yarn/TestSamzaYarnAppMasterLifecycle.scala
+++ b/samza-yarn/src/test/scala/org/apache/samza/job/yarn/TestSamzaYarnAppMasterLifecycle.scala
@@ -54,13 +54,15 @@ class TestSamzaYarnAppMasterLifecycle {
new RegisterApplicationMasterResponse {
override def setApplicationACLs(map: java.util.Map[ApplicationAccessType, String]): Unit = ()
override def getApplicationACLs = null
+ override def setResourceTypes(types: java.util.List[org.apache.hadoop.yarn.api.records.ResourceTypeInfo]): Unit = ()
+ override def getResourceTypes = null
override def setMaximumResourceCapability(r: Resource): Unit = ()
override def getMaximumResourceCapability = new Resource {
def getMemory = 512
def getVirtualCores = 2
def setMemory(memory: Int) {}
def setVirtualCores(vCores: Int) {}
- def compareTo(o: Resource) = 0
+ override def compareTo(o: Resource) = 0
}
override def getClientToAMTokenMasterKey = null
override def setClientToAMTokenMasterKey(buffer: ByteBuffer) {}