You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2014/05/22 15:08:00 UTC
git commit: AMBARI-5797. Agent function get_port_from_url is broken
(aonishuk)
Repository: ambari
Updated Branches:
refs/heads/trunk 1423a43d0 -> fd9740312
AMBARI-5797. Agent function get_port_from_url is broken (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fd974031
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fd974031
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fd974031
Branch: refs/heads/trunk
Commit: fd97403125bf584459ce900ffcb7e7fc208b28d7
Parents: 1423a43
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu May 22 16:07:50 2014 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu May 22 16:07:50 2014 +0300
----------------------------------------------------------------------
.../libraries/functions/get_port_from_url.py | 13 ++++++--
.../resource_management/TestLibraryFunctions.py | 33 ++++++++++++++++++++
.../1.3.2/configs/default.hbasedecom.json | 4 +--
.../python/stacks/1.3.2/configs/default.json | 2 +-
.../1.3.2/configs/default.non_gmetad_host.json | 2 +-
.../python/stacks/1.3.2/configs/secured.json | 2 +-
.../1.3.2/configs/secured_no_jce_name.json | 2 +-
.../2.0.6/configs/default.hbasedecom.json | 2 +-
.../python/stacks/2.0.6/configs/default.json | 2 +-
.../2.0.6/configs/default.non_gmetad_host.json | 2 +-
.../stacks/2.0.6/configs/flume_target.json | 2 +-
.../python/stacks/2.0.6/configs/ha_default.json | 2 +-
.../python/stacks/2.0.6/configs/ha_secured.json | 2 +-
.../python/stacks/2.0.6/configs/secured.json | 2 +-
.../2.0.6/configs/secured_no_jce_name.json | 2 +-
.../test/python/stacks/2.1/configs/default.json | 2 +-
.../test/python/stacks/2.1/configs/secured.json | 2 +-
17 files changed, 60 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/fd974031/ambari-agent/src/main/python/resource_management/libraries/functions/get_port_from_url.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/libraries/functions/get_port_from_url.py b/ambari-agent/src/main/python/resource_management/libraries/functions/get_port_from_url.py
index ce9185b..70bd2d7 100644
--- a/ambari-agent/src/main/python/resource_management/libraries/functions/get_port_from_url.py
+++ b/ambari-agent/src/main/python/resource_management/libraries/functions/get_port_from_url.py
@@ -22,10 +22,19 @@ Ambari Agent
from resource_management import *
from resource_management.libraries.functions.is_empty import *
-from urlparse import urlparse
+from resource_management.core.exceptions import Fail
+import re
def get_port_from_url(address):
+ """
+ Return port from URL. If address is UnknownConfiguration,
+ UnknownConfiguration will be returned. If no port was found, Fail will be
+ raised.
+ """
if not is_empty(address):
- return urlparse(address).port
+ port = re.findall(":([\d]{1,5})(?=/|$)", address)
+ if port:
+ return port[0]
+ raise Fail("No port in URL:{0}".format(address))
else:
return address
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/fd974031/ambari-agent/src/test/python/resource_management/TestLibraryFunctions.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestLibraryFunctions.py b/ambari-agent/src/test/python/resource_management/TestLibraryFunctions.py
new file mode 100644
index 0000000..4e6b6c3
--- /dev/null
+++ b/ambari-agent/src/test/python/resource_management/TestLibraryFunctions.py
@@ -0,0 +1,33 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+from resource_management.libraries.functions import get_port_from_url
+from resource_management.core.exceptions import Fail
+class TestLibraryFunctions(TestCase):
+
+ def test_get_port_from_url(self):
+ self.assertEqual("8080",get_port_from_url("protocol://host:8080"))
+ self.assertEqual("8080",get_port_from_url("protocol://host:8080/"))
+ self.assertEqual("8080",get_port_from_url("host:8080"))
+ self.assertEqual("8080",get_port_from_url("host:8080/"))
+ self.assertEqual("8080",get_port_from_url("host:8080/dots_in_url8888:"))
+ self.assertEqual("8080",get_port_from_url("protocol://host:8080/dots_in_url8888:"))
+ self.assertEqual("8080",get_port_from_url("127.0.0.1:8080"))
+ self.assertRaises(Fail, get_port_from_url, "http://host/no_port")
+ self.assertRaises(Fail, get_port_from_url, "127.0.0.1:808080")
http://git-wip-us.apache.org/repos/asf/ambari/blob/fd974031/ambari-server/src/test/python/stacks/1.3.2/configs/default.hbasedecom.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/configs/default.hbasedecom.json b/ambari-server/src/test/python/stacks/1.3.2/configs/default.hbasedecom.json
index ce92689..8732c0b 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/configs/default.hbasedecom.json
+++ b/ambari-server/src/test/python/stacks/1.3.2/configs/default.hbasedecom.json
@@ -233,11 +233,11 @@
"dfs.datanode.du.reserved": "1073741824",
"dfs.webhdfs.enabled": "true",
"dfs.namenode.handler.count": "100",
- "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}",
+ "dfs.datanode.http.address": "0.0.0.0:50075",
"dfs.datanode.socket.write.timeout": "0",
"ipc.server.read.threadpool.size": "5",
"dfs.balance.bandwidthPerSec": "6250000",
- "dfs.datanode.address": "0.0.0.0:${ambari.dfs.datanode.port}",
+ "dfs.datanode.address": "0.0.0.0:50075",
"dfs.blockreport.initialDelay": "120",
"dfs.datanode.failed.volumes.tolerated": "0",
"dfs.permissions.supergroup": "hdfs",
http://git-wip-us.apache.org/repos/asf/ambari/blob/fd974031/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/configs/default.json b/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
index 8eabd30..a56eeba 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
+++ b/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
@@ -235,7 +235,7 @@
"dfs.datanode.du.reserved": "1073741824",
"dfs.webhdfs.enabled": "true",
"dfs.namenode.handler.count": "100",
- "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}",
+ "dfs.datanode.http.address": "0.0.0.0:50075",
"dfs.datanode.socket.write.timeout": "0",
"ipc.server.read.threadpool.size": "5",
"dfs.balance.bandwidthPerSec": "6250000",
http://git-wip-us.apache.org/repos/asf/ambari/blob/fd974031/ambari-server/src/test/python/stacks/1.3.2/configs/default.non_gmetad_host.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/configs/default.non_gmetad_host.json b/ambari-server/src/test/python/stacks/1.3.2/configs/default.non_gmetad_host.json
index d9b928d..376b59d 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/configs/default.non_gmetad_host.json
+++ b/ambari-server/src/test/python/stacks/1.3.2/configs/default.non_gmetad_host.json
@@ -233,7 +233,7 @@
"dfs.datanode.du.reserved": "1073741824",
"dfs.webhdfs.enabled": "true",
"dfs.namenode.handler.count": "100",
- "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}",
+ "dfs.datanode.http.address": "0.0.0.0:50075",
"dfs.datanode.socket.write.timeout": "0",
"ipc.server.read.threadpool.size": "5",
"dfs.balance.bandwidthPerSec": "6250000",
http://git-wip-us.apache.org/repos/asf/ambari/blob/fd974031/ambari-server/src/test/python/stacks/1.3.2/configs/secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/configs/secured.json b/ambari-server/src/test/python/stacks/1.3.2/configs/secured.json
index fe3d6f6..c0a345e 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/configs/secured.json
+++ b/ambari-server/src/test/python/stacks/1.3.2/configs/secured.json
@@ -331,7 +331,7 @@
"dfs.access.time.precision": "0",
"dfs.secondary.namenode.kerberos.internal.spnego.principal": "${dfs.web.authentication.kerberos.principal}",
"dfs.https.address": "c6401.ambari.apache.org:50470",
- "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}",
+ "dfs.datanode.http.address": "0.0.0.0:1022",
"dfs.data.dir": "/hadoop/hdfs/data",
"dfs.secondary.https.port": "50490",
"dfs.permissions": "true",
http://git-wip-us.apache.org/repos/asf/ambari/blob/fd974031/ambari-server/src/test/python/stacks/1.3.2/configs/secured_no_jce_name.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/configs/secured_no_jce_name.json b/ambari-server/src/test/python/stacks/1.3.2/configs/secured_no_jce_name.json
index 105d0b7..677a759 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/configs/secured_no_jce_name.json
+++ b/ambari-server/src/test/python/stacks/1.3.2/configs/secured_no_jce_name.json
@@ -329,7 +329,7 @@
"dfs.access.time.precision": "0",
"dfs.secondary.namenode.kerberos.internal.spnego.principal": "${dfs.web.authentication.kerberos.principal}",
"dfs.https.address": "c6401.ambari.apache.org:50470",
- "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}",
+ "dfs.datanode.http.address": "0.0.0.0:1022",
"dfs.data.dir": "/hadoop/hdfs/data",
"dfs.secondary.https.port": "50490",
"dfs.permissions": "true",
http://git-wip-us.apache.org/repos/asf/ambari/blob/fd974031/ambari-server/src/test/python/stacks/2.0.6/configs/default.hbasedecom.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.hbasedecom.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.hbasedecom.json
index 4817a58..553cc59 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.hbasedecom.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.hbasedecom.json
@@ -331,7 +331,7 @@
"dfs.namenode.handler.count": "100",
"dfs.namenode.checkpoint.dir": "/hadoop/hdfs/namesecondary",
"fs.permissions.umask-mode": "022",
- "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}",
+ "dfs.datanode.http.address": "0.0.0.0:50075",
"dfs.datanode.ipc.address": "0.0.0.0:8010",
"dfs.datanode.data.dir": "/hadoop/hdfs/data",
"dfs.namenode.http-address": "c6401.ambari.apache.org:50070",
http://git-wip-us.apache.org/repos/asf/ambari/blob/fd974031/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
index f2d40e9..8f6d64b 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
@@ -334,7 +334,7 @@
"dfs.namenode.handler.count": "100",
"dfs.namenode.checkpoint.dir": "/hadoop/hdfs/namesecondary",
"fs.permissions.umask-mode": "022",
- "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}",
+ "dfs.datanode.http.address": "0.0.0.0:50075",
"dfs.datanode.ipc.address": "0.0.0.0:8010",
"dfs.datanode.data.dir": "/hadoop/hdfs/data",
"dfs.namenode.http-address": "c6401.ambari.apache.org:50070",
http://git-wip-us.apache.org/repos/asf/ambari/blob/fd974031/ambari-server/src/test/python/stacks/2.0.6/configs/default.non_gmetad_host.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.non_gmetad_host.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.non_gmetad_host.json
index b30714c..51908f4 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.non_gmetad_host.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.non_gmetad_host.json
@@ -332,7 +332,7 @@
"dfs.namenode.handler.count": "100",
"dfs.namenode.checkpoint.dir": "/hadoop/hdfs/namesecondary",
"fs.permissions.umask-mode": "022",
- "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}",
+ "dfs.datanode.http.address": "0.0.0.0:50075",
"dfs.datanode.ipc.address": "0.0.0.0:8010",
"dfs.datanode.data.dir": "/hadoop/hdfs/data",
"dfs.namenode.http-address": "c6401.ambari.apache.org:50070",
http://git-wip-us.apache.org/repos/asf/ambari/blob/fd974031/ambari-server/src/test/python/stacks/2.0.6/configs/flume_target.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/flume_target.json b/ambari-server/src/test/python/stacks/2.0.6/configs/flume_target.json
index 931f202..a50b24a 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/flume_target.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/flume_target.json
@@ -333,7 +333,7 @@
"dfs.namenode.handler.count": "100",
"dfs.namenode.checkpoint.dir": "/hadoop/hdfs/namesecondary",
"fs.permissions.umask-mode": "022",
- "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}",
+ "dfs.datanode.http.address": "0.0.0.0:50075",
"dfs.datanode.ipc.address": "0.0.0.0:8010",
"dfs.datanode.data.dir": "/hadoop/hdfs/data",
"dfs.namenode.http-address": "c6401.ambari.apache.org:50070",
http://git-wip-us.apache.org/repos/asf/ambari/blob/fd974031/ambari-server/src/test/python/stacks/2.0.6/configs/ha_default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/ha_default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/ha_default.json
index 0ea0db3..319d230 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/ha_default.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/ha_default.json
@@ -178,7 +178,7 @@
"dfs.namenode.write.stale.datanode.ratio": "1.0f",
"dfs.namenode.secondary.http-address": "c6402.ambari.apache.org:50090",
"dfs.ha.fencing.methods": "shell(/bin/true)",
- "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}",
+ "dfs.datanode.http.address": "0.0.0.0:50075",
"dfs.datanode.du.reserved": "1073741824",
"dfs.client.read.shortcircuit.streams.cache.size": "4096",
"dfs.ha.namenodes.ns1": "nn1,nn2",
http://git-wip-us.apache.org/repos/asf/ambari/blob/fd974031/ambari-server/src/test/python/stacks/2.0.6/configs/ha_secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/ha_secured.json b/ambari-server/src/test/python/stacks/2.0.6/configs/ha_secured.json
index dce20fa..14c98a3 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/ha_secured.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/ha_secured.json
@@ -248,7 +248,7 @@
"dfs.datanode.kerberos.principal": "dn/_HOST@EXAMPLE.COM",
"dfs.ha.fencing.methods": "shell(/bin/true)",
"dfs.journalnode.keytab.file": "/etc/security/keytabs/jn.service.keytab",
- "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}",
+ "dfs.datanode.http.address": "0.0.0.0:1022",
"dfs.datanode.du.reserved": "1073741824",
"dfs.client.read.shortcircuit.streams.cache.size": "4096",
"dfs.namenode.rpc-address.ns1.nn2": "c6402.ambari.apache.org:8020",
http://git-wip-us.apache.org/repos/asf/ambari/blob/fd974031/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
index d594eea..829606b 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
@@ -446,7 +446,7 @@
"dfs.namenode.secondary.http-address": "c6402.ambari.apache.org:50090",
"dfs.client.read.shortcircuit": "true",
"dfs.journalnode.keytab.file": "/etc/security/keytabs/jn.service.keytab",
- "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}",
+ "dfs.datanode.http.address": "0.0.0.0:1022",
"dfs.datanode.du.reserved": "1073741824",
"dfs.client.read.shortcircuit.streams.cache.size": "4096",
"dfs.secondary.namenode.keytab.file": "/etc/security/keytabs/nn.service.keytab",
http://git-wip-us.apache.org/repos/asf/ambari/blob/fd974031/ambari-server/src/test/python/stacks/2.0.6/configs/secured_no_jce_name.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/secured_no_jce_name.json b/ambari-server/src/test/python/stacks/2.0.6/configs/secured_no_jce_name.json
index ecf5031..a4d46ae 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/secured_no_jce_name.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/secured_no_jce_name.json
@@ -444,7 +444,7 @@
"dfs.namenode.secondary.http-address": "c6402.ambari.apache.org:50090",
"dfs.client.read.shortcircuit": "true",
"dfs.journalnode.keytab.file": "/etc/security/keytabs/jn.service.keytab",
- "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}",
+ "dfs.datanode.http.address": "0.0.0.0:1022",
"dfs.datanode.du.reserved": "1073741824",
"dfs.client.read.shortcircuit.streams.cache.size": "4096",
"dfs.secondary.namenode.keytab.file": "/etc/security/keytabs/nn.service.keytab",
http://git-wip-us.apache.org/repos/asf/ambari/blob/fd974031/ambari-server/src/test/python/stacks/2.1/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/configs/default.json b/ambari-server/src/test/python/stacks/2.1/configs/default.json
index 50c72ea..b4294ce 100644
--- a/ambari-server/src/test/python/stacks/2.1/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.1/configs/default.json
@@ -372,7 +372,7 @@
"dfs.namenode.handler.count": "100",
"dfs.namenode.checkpoint.dir": "/hadoop/hdfs/namesecondary",
"fs.permissions.umask-mode": "022",
- "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}",
+ "dfs.datanode.http.address": "0.0.0.0:50075",
"dfs.datanode.ipc.address": "0.0.0.0:8010",
"dfs.datanode.data.dir": "/hadoop/hdfs/data",
"dfs.namenode.http-address": "c6401.ambari.apache.org:50070",
http://git-wip-us.apache.org/repos/asf/ambari/blob/fd974031/ambari-server/src/test/python/stacks/2.1/configs/secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/configs/secured.json b/ambari-server/src/test/python/stacks/2.1/configs/secured.json
index 3dfeb0f..430e0be 100644
--- a/ambari-server/src/test/python/stacks/2.1/configs/secured.json
+++ b/ambari-server/src/test/python/stacks/2.1/configs/secured.json
@@ -494,7 +494,7 @@
"dfs.namenode.secondary.http-address": "c6402.ambari.apache.org:50090",
"dfs.client.read.shortcircuit": "true",
"dfs.journalnode.keytab.file": "/etc/security/keytabs/jn.service.keytab",
- "dfs.datanode.http.address": "0.0.0.0:${ambari.dfs.datanode.http.port}",
+ "dfs.datanode.http.address": "0.0.0.0:1022",
"dfs.datanode.du.reserved": "1073741824",
"dfs.client.read.shortcircuit.streams.cache.size": "4096",
"dfs.secondary.namenode.keytab.file": "/etc/security/keytabs/nn.service.keytab",