You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by ya...@apache.org on 2015/06/10 00:51:30 UTC
samza git commit: SAMZA-688: added https support for YarnJobFactory
Repository: samza
Updated Branches:
refs/heads/master 69b1f2e56 -> 0f8d7d121
SAMZA-688: added https support for YarnJobFactory
Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/0f8d7d12
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/0f8d7d12
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/0f8d7d12
Branch: refs/heads/master
Commit: 0f8d7d121f4a4d21dff5dfb58c1f856a192b04b1
Parents: 69b1f2e
Author: Jose <jo...@stormpath.com>
Authored: Tue Jun 9 15:51:55 2015 -0700
Committer: Yan Fang <ya...@gmail.com>
Committed: Tue Jun 9 15:51:55 2015 -0700
----------------------------------------------------------------------
.../apache/samza/job/yarn/YarnJobFactory.scala | 1 +
.../samza/job/yarn/TestYarnJobFactory.scala | 45 ++++++++++++++++++++
2 files changed, 46 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/samza/blob/0f8d7d12/samza-yarn/src/main/scala/org/apache/samza/job/yarn/YarnJobFactory.scala
----------------------------------------------------------------------
diff --git a/samza-yarn/src/main/scala/org/apache/samza/job/yarn/YarnJobFactory.scala b/samza-yarn/src/main/scala/org/apache/samza/job/yarn/YarnJobFactory.scala
index a513567..4e328a5 100644
--- a/samza-yarn/src/main/scala/org/apache/samza/job/yarn/YarnJobFactory.scala
+++ b/samza-yarn/src/main/scala/org/apache/samza/job/yarn/YarnJobFactory.scala
@@ -28,6 +28,7 @@ class YarnJobFactory extends StreamJobFactory {
// TODO fix this. needed to support http package locations.
val hConfig = new YarnConfiguration
hConfig.set("fs.http.impl", classOf[HttpFileSystem].getName)
+ hConfig.set("fs.https.impl", classOf[HttpFileSystem].getName)
new YarnJob(config, hConfig)
}
http://git-wip-us.apache.org/repos/asf/samza/blob/0f8d7d12/samza-yarn/src/test/scala/org/apache/samza/job/yarn/TestYarnJobFactory.scala
----------------------------------------------------------------------
diff --git a/samza-yarn/src/test/scala/org/apache/samza/job/yarn/TestYarnJobFactory.scala b/samza-yarn/src/test/scala/org/apache/samza/job/yarn/TestYarnJobFactory.scala
new file mode 100644
index 0000000..110c0fc
--- /dev/null
+++ b/samza-yarn/src/test/scala/org/apache/samza/job/yarn/TestYarnJobFactory.scala
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.samza.job.yarn
+
+import org.apache.samza.config.MapConfig
+import org.apache.samza.util.hadoop.HttpFileSystem
+import org.junit.Assert._
+import org.junit.Test
+
+
+class TestYarnJobFactory {
+
+ @Test
+ def testGetJob {
+
+ val jobFactory = new YarnJobFactory
+
+ val yarnJob = jobFactory.getJob(new MapConfig)
+
+ val hConfig = yarnJob.client.yarnClient.getConfig
+
+ assertEquals(classOf[HttpFileSystem].getName, hConfig.get("fs.http.impl"))
+
+ assertEquals(classOf[HttpFileSystem].getName, hConfig.get("fs.https.impl"))
+
+ }
+}
+