You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ambari.apache.org by "Andrew Onischuk (JIRA)" <ji...@apache.org> on 2016/10/05 10:04:20 UTC
[jira] [Created] (AMBARI-18536) Components can't be started because
of failed webhdfs request
Andrew Onischuk created AMBARI-18536:
----------------------------------------
Summary: Components can't be started because of failed webhdfs request
Key: AMBARI-18536
URL: https://issues.apache.org/jira/browse/AMBARI-18536
Project: Ambari
Issue Type: Bug
Reporter: Andrew Onischuk
Assignee: Andrew Onischuk
Fix For: 2.5.0
Attachments: AMBARI-18536.patch
Different components fail to start, for instance:
Traceback (most recent call last):\n File \"/var/lib/ambari-agent/cache/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py\", line 182, in <module>\n HistoryServer().execute()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/script/script.py\", line 219, in execute\n method(env)\n File \"/var/lib/ambari-agent/cache/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py\", line 100, in start\n host_sys_prepped=params.host_sys_prepped)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/functions/copy_tarball.py\", line 204, in copy_to_hdfs\n replace_existing_files=replace_existing_files,\n File \"/usr/lib/python2.6/site-packages/resource_management/core/base.py\", line 154, in __init__\n self.env.run()\n File \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", line 160, in run\n self.run_action(resource, action)\n File \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", line 124, in run_action\n provider_action()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 463, in action_create_on_execute\n self.action_delayed(\"create\")\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 460, in action_delayed\n self.get_hdfs_resource_executor().action_delayed(action_name, self)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 258, in action_delayed\n self._create_resource()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 272, in _create_resource\n self._create_file(self.main_resource.resource.target, source=self.main_resource.resource.source, mode=self.mode)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 325, in _create_file\n self.util.run_command(target, 'CREATE', method='PUT', overwrite=True, assertable_result=False, file_to_put=source, **kwargs)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 182, in run_command\n _, out, err = get_user_call_output(cmd, user=self.run_user, logoutput=self.logoutput, quiet=False)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/functions/get_user_call_output.py\", line 61, in get_user_call_output\n raise Fail(err_msg)\nresource_management.core.exceptions.Fail: Execution of 'curl -sS -L -w '%{http_code}' -X PUT --data-binary @/usr/hdp/2.4.3.0-227/hadoop/mapreduce.tar.gz --negotiate -u : 'http://nat-r6-tzrs-ambari-alerts-1.openstacklocal:50070/webhdfs/v1/hdp/apps/2.4.3.0-227/mapreduce/mapreduce.tar.gz?op=CREATE&user.name=cstm-hdfs&overwrite=True&permission=444' 1>/tmp/tmpNxSlRa 2>/tmp/tmpXlLRgB' returned 52. curl: (52) Empty reply from server\n100
Traceback (most recent call last):\n File \"/var/lib/ambari-agent/cache/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py\", line 165, in <module>\n FalconServer().execute()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/script/script.py\", line 219, in execute\n method(env)\n File \"/var/lib/ambari-agent/cache/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py\", line 46, in start\n self.configure(env, upgrade_type=upgrade_type)\n File \"/var/lib/ambari-agent/cache/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py\", line 41, in configure\n falcon('server', action='config', upgrade_type=upgrade_type)\n File \"/usr/lib/python2.6/site-packages/ambari_commons/os_family_impl.py\", line 89, in thunk\n return fn(*args, **kwargs)\n File \"/var/lib/ambari-agent/cache/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py\", line 127, in falcon\n mode = 0777)\n File \"/usr/lib/python2.6/site-packages/resource_management/core/base.py\", line 154, in __init__\n self.env.run()\n File \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", line 160, in run\n self.run_action(resource, action)\n File \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", line 124, in run_action\n provider_action()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 463, in action_create_on_execute\n self.action_delayed(\"create\")\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 460, in action_delayed\n self.get_hdfs_resource_executor().action_delayed(action_name, self)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 250, in action_delayed\n self._assert_valid()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 234, in _assert_valid\n self.target_status = self._get_file_status(target)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 295, in _get_file_status\n list_status = self.util.run_command(target, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 182, in run_command\n _, out, err = get_user_call_output(cmd, user=self.run_user, logoutput=self.logoutput, quiet=False)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/functions/get_user_call_output.py\", line 61, in get_user_call_output\n raise Fail(err_msg)\nresource_management.core.exceptions.Fail: Execution of 'curl -sS -L -w '%{http_code}' -X GET --negotiate -u : 'http://nat-r6-tzrs-ambari-alerts-1.openstacklocal:50070/webhdfs/v1/apps/falcon?op=GETFILESTATUS&user.name=cstm-hdfs' 1>/tmp/tmp2adaiW 2>/tmp/tmpEu_LF7' returned 7. curl: (7) couldn't connect to host\n000
Traceback (most recent call last):\n File \"/var/lib/ambari-agent/cache/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py\", line 149, in <module>\n HbaseMaster().execute()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/script/script.py\", line 219, in execute\n method(env)\n File \"/var/lib/ambari-agent/cache/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py\", line 83, in start\n self.configure(env) # for security\n File \"/var/lib/ambari-agent/cache/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_master.py\", line 39, in configure\n hbase(name='master')\n File \"/usr/lib/python2.6/site-packages/ambari_commons/os_family_impl.py\", line 89, in thunk\n return fn(*args, **kwargs)\n File \"/var/lib/ambari-agent/cache/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py\", line 184, in hbase\n owner=params.hbase_user\n File \"/usr/lib/python2.6/site-packages/resource_management/core/base.py\", line 154, in __init__\n self.env.run()\n File \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", line 160, in run\n self.run_action(resource, action)\n File \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", line 124, in run_action\n provider_action()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 463, in action_create_on_execute\n self.action_delayed(\"create\")\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 460, in action_delayed\n self.get_hdfs_resource_executor().action_delayed(action_name, self)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 250, in action_delayed\n self._assert_valid()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 234, in _assert_valid\n self.target_status = self._get_file_status(target)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 295, in _get_file_status\n list_status = self.util.run_command(target, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 182, in run_command\n _, out, err = get_user_call_output(cmd, user=self.run_user, logoutput=self.logoutput, quiet=False)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/functions/get_user_call_output.py\", line 61, in get_user_call_output\n raise Fail(err_msg)\nresource_management.core.exceptions.Fail: Execution of 'curl -sS -L -w '%{http_code}' -X GET --negotiate -u : 'http://nat-r6-tzrs-ambari-alerts-1.openstacklocal:50070/webhdfs/v1/apps/hbase/data?op=GETFILESTATUS&user.name=cstm-hdfs' 1>/tmp/tmp7NmmTu 2>/tmp/tmpHeD3Y8' returned 7. curl: (7) couldn't connect to host\n000
Traceback (most recent call last):\n File \"/var/lib/ambari-agent/cache/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py\", line 203, in <module>\n HiveServer().execute()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/script/script.py\", line 219, in execute\n method(env)\n File \"/var/lib/ambari-agent/cache/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py\", line 84, in start\n self.configure(env) # FOR SECURITY\n File \"/var/lib/ambari-agent/cache/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py\", line 55, in configure\n hive(name='hiveserver2')\n File \"/usr/lib/python2.6/site-packages/ambari_commons/os_family_impl.py\", line 89, in thunk\n return fn(*args, **kwargs)\n File \"/var/lib/ambari-agent/cache/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py\", line 124, in hive\n mode=params.hcat_hdfs_user_mode\n File \"/usr/lib/python2.6/site-packages/resource_management/core/base.py\", line 154, in __init__\n self.env.run()\n File \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", line 160, in run\n self.run_action(resource, action)\n File \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", line 124, in run_action\n provider_action()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 463, in action_create_on_execute\n self.action_delayed(\"create\")\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 460, in action_delayed\n self.get_hdfs_resource_executor().action_delayed(action_name, self)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 250, in action_delayed\n self._assert_valid()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 234, in _assert_valid\n self.target_status = self._get_file_status(target)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 295, in _get_file_status\n list_status = self.util.run_command(target, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 182, in run_command\n _, out, err = get_user_call_output(cmd, user=self.run_user, logoutput=self.logoutput, quiet=False)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/functions/get_user_call_output.py\", line 61, in get_user_call_output\n raise Fail(err_msg)\nresource_management.core.exceptions.Fail: Execution of 'curl -sS -L -w '%{http_code}' -X GET --negotiate -u : 'http://nat-r6-tzrs-ambari-alerts-1.openstacklocal:50070/webhdfs/v1/user/hcat?op=GETFILESTATUS&user.name=cstm-hdfs' 1>/tmp/tmpQxm4XF 2>/tmp/tmpfD0pbF' returned 7. curl: (7) couldn't connect to host\n000
Traceback (most recent call last):\n File \"/var/lib/ambari-agent/cache/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py\", line 210, in <module>\n OozieServer().execute()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/script/script.py\", line 219, in execute\n method(env)\n File \"/var/lib/ambari-agent/cache/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py\", line 88, in start\n self.configure(env)\n File \"/var/lib/ambari-agent/cache/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py\", line 82, in configure\n oozie(is_server=True)\n File \"/usr/lib/python2.6/site-packages/ambari_commons/os_family_impl.py\", line 89, in thunk\n return fn(*args, **kwargs)\n File \"/var/lib/ambari-agent/cache/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py\", line 99, in oozie\n mode=params.oozie_hdfs_user_mode\n File \"/usr/lib/python2.6/site-packages/resource_management/core/base.py\", line 154, in __init__\n self.env.run()\n File \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", line 160, in run\n self.run_action(resource, action)\n File \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", line 124, in run_action\n provider_action()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 463, in action_create_on_execute\n self.action_delayed(\"create\")\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 460, in action_delayed\n self.get_hdfs_resource_executor().action_delayed(action_name, self)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 250, in action_delayed\n self._assert_valid()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 234, in _assert_valid\n self.target_status = self._get_file_status(target)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 295, in _get_file_status\n list_status = self.util.run_command(target, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 182, in run_command\n _, out, err = get_user_call_output(cmd, user=self.run_user, logoutput=self.logoutput, quiet=False)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/functions/get_user_call_output.py\", line 61, in get_user_call_output\n raise Fail(err_msg)\nresource_management.core.exceptions.Fail: Execution of 'curl -sS -L -w '%{http_code}' -X GET --negotiate -u : 'http://nat-r6-tzrs-ambari-alerts-1.openstacklocal:50070/webhdfs/v1/user/cstm-oozie?op=GETFILESTATUS&user.name=cstm-hdfs' 1>/tmp/tmpG5odxb 2>/tmp/tmp9KV3kC' returned 7. curl: (7) couldn't connect to host\n000
Traceback (most recent call last):\n File \"/var/lib/ambari-agent/cache/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py\", line 95, in <module>\n JobHistoryServer().execute()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/script/script.py\", line 219, in execute\n method(env)\n File \"/var/lib/ambari-agent/cache/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py\", line 54, in start\n self.configure(env)\n File \"/var/lib/ambari-agent/cache/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py\", line 48, in configure\n setup_spark(env, 'server', upgrade_type=upgrade_type, action = 'config')\n File \"/var/lib/ambari-agent/cache/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py\", line 46, in setup_spark\n mode=0775\n File \"/usr/lib/python2.6/site-packages/resource_management/core/base.py\", line 154, in __init__\n self.env.run()\n File \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", line 160, in run\n self.run_action(resource, action)\n File \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", line 124, in run_action\n provider_action()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 463, in action_create_on_execute\n self.action_delayed(\"create\")\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 460, in action_delayed\n self.get_hdfs_resource_executor().action_delayed(action_name, self)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 250, in action_delayed\n self._assert_valid()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 234, in _assert_valid\n self.target_status = self._get_file_status(target)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 295, in _get_file_status\n list_status = self.util.run_command(target, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 182, in run_command\n _, out, err = get_user_call_output(cmd, user=self.run_user, logoutput=self.logoutput, quiet=False)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/functions/get_user_call_output.py\", line 61, in get_user_call_output\n raise Fail(err_msg)\nresource_management.core.exceptions.Fail: Execution of 'curl -sS -L -w '%{http_code}' -X GET --negotiate -u : 'http://nat-r6-tzrs-ambari-alerts-1.openstacklocal:50070/webhdfs/v1/user/cstm-spark?op=GETFILESTATUS&user.name=cstm-hdfs' 1>/tmp/tmpDcu_eG 2>/tmp/tmpdF157D' returned 7. curl: (7) couldn't connect to host\n000
Traceback (most recent call last):\n File \"/var/lib/ambari-agent/cache/common-services/SPARK/1.2.0.2.2/package/scripts/spark_thrift_server.py\", line 79, in <module>\n SparkThriftServer().execute()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/script/script.py\", line 219, in execute\n method(env)\n File \"/var/lib/ambari-agent/cache/common-services/SPARK/1.2.0.2.2/package/scripts/spark_thrift_server.py\", line 53, in start\n self.configure(env)\n File \"/var/lib/ambari-agent/cache/common-services/SPARK/1.2.0.2.2/package/scripts/spark_thrift_server.py\", line 47, in configure\n setup_spark(env, 'server', upgrade_type = upgrade_type, action = 'config')\n File \"/var/lib/ambari-agent/cache/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py\", line 46, in setup_spark\n mode=0775\n File \"/usr/lib/python2.6/site-packages/resource_management/core/base.py\", line 154, in __init__\n self.env.run()\n File \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", line 160, in run\n self.run_action(resource, action)\n File \"/usr/lib/python2.6/site-packages/resource_management/core/environment.py\", line 124, in run_action\n provider_action()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 463, in action_create_on_execute\n self.action_delayed(\"create\")\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 460, in action_delayed\n self.get_hdfs_resource_executor().action_delayed(action_name, self)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 250, in action_delayed\n self._assert_valid()\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 234, in _assert_valid\n self.target_status = self._get_file_status(target)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 295, in _get_file_status\n list_status = self.util.run_command(target, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/providers/hdfs_resource.py\", line 182, in run_command\n _, out, err = get_user_call_output(cmd, user=self.run_user, logoutput=self.logoutput, quiet=False)\n File \"/usr/lib/python2.6/site-packages/resource_management/libraries/functions/get_user_call_output.py\", line 61, in get_user_call_output\n raise Fail(err_msg)\nresource_management.core.exceptions.Fail: Execution of 'curl -sS -L -w '%{http_code}' -X GET --negotiate -u : 'http://nat-r6-tzrs-ambari-alerts-1.openstacklocal:50070/webhdfs/v1/user/cstm-spark?op=GETFILESTATUS&user.name=cstm-hdfs' 1>/tmp/tmpL8hR95 2>/tmp/tmpwE5KZf' returned 7. curl: (7) couldn't connect to host\n000
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)