You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by to...@apache.org on 2013/10/21 22:31:14 UTC
[1/2] git commit: Add log statements to our ParamikoSSHClient
wrapper. This should make debugging deployment issues easier.
Updated Branches:
refs/heads/trunk 0e7341019 -> d4ab3ab90
Add log statements to our ParamikoSSHClient wrapper. This should make
debugging deployment issues easier.
Part of LIBCLOUD-414.
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/df6dc611
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/df6dc611
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/df6dc611
Branch: refs/heads/trunk
Commit: df6dc6119003c7d6ebe3bf3a237ad4d9d56982d1
Parents: 0e73410
Author: Tomaz Muraus <to...@apache.org>
Authored: Mon Oct 21 22:17:48 2013 +0200
Committer: Tomaz Muraus <to...@apache.org>
Committed: Mon Oct 21 22:17:48 2013 +0200
----------------------------------------------------------------------
CHANGES | 8 +++++
libcloud/compute/ssh.py | 45 ++++++++++++++++++-------
libcloud/test/compute/test_ssh_client.py | 22 +++++++++++--
libcloud/utils/logging.py | 47 +++++++++++++++++++++++++++
4 files changed, 109 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/df6dc611/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 8cc1449..fe8db26 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,5 +1,13 @@
-*- coding: utf-8 -*-
+Changes with Apache Libcloud in development
+
+ *) Compute
+
+ - Add log statements to our ParamikoSSHClient wrapper. This should make
+ debugging deployment issues easier. (LIBCLOUD-414)
+ [Tomaz Muraus]
+
Changes with Apache Libcloud 0.14.0-beta1
*) General
http://git-wip-us.apache.org/repos/asf/libcloud/blob/df6dc611/libcloud/compute/ssh.py
----------------------------------------------------------------------
diff --git a/libcloud/compute/ssh.py b/libcloud/compute/ssh.py
index bc23e4d..d476e87 100644
--- a/libcloud/compute/ssh.py
+++ b/libcloud/compute/ssh.py
@@ -35,6 +35,8 @@ import logging
from os.path import split as psplit
from os.path import join as pjoin
+from libcloud.utils.logging import ExtraLogFormatter
+
class BaseSSHClient(object):
"""
@@ -136,6 +138,18 @@ class BaseSSHClient(object):
raise NotImplementedError(
'close not implemented for this ssh client')
+ def _get_and_setup_logger(self):
+ logger = logging.getLogger('libcloud.compute.ssh')
+ path = os.getenv('LIBCLOUD_DEBUG')
+
+ if path:
+ handler = logging.FileHandler(path)
+ handler.setFormatter(ExtraLogFormatter())
+ logger.addHandler(handler)
+ logger.setLevel(logging.DEBUG)
+
+ return logger
+
class ParamikoSSHClient(BaseSSHClient):
@@ -148,6 +162,7 @@ class ParamikoSSHClient(BaseSSHClient):
password, key, timeout)
self.client = paramiko.SSHClient()
self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ self.logger = self._get_and_setup_logger()
def connect(self):
conninfo = {'hostname': self.hostname,
@@ -167,10 +182,17 @@ class ParamikoSSHClient(BaseSSHClient):
if self.timeout:
conninfo['timeout'] = self.timeout
+ extra = {'_hostname': self.hostname, '_port': self.port,
+ '_username': self.username, '_timeout': self.timeout}
+ self.logger.debug('Connecting to server', extra=extra)
+
self.client.connect(**conninfo)
return True
def put(self, path, contents=None, chmod=None, mode='w'):
+ extra = {'_path': path, '_mode': mode, '_chmod': chmod}
+ self.logger.debug('Uploading file', extra=extra)
+
sftp = self.client.open_sftp()
# less than ideal, but we need to mkdir stuff otherwise file() fails
head, tail = psplit(path)
@@ -208,12 +230,18 @@ class ParamikoSSHClient(BaseSSHClient):
return file_path
def delete(self, path):
+ extra = {'_path': path}
+ self.logger.debug('Deleting file', extra=extra)
+
sftp = self.client.open_sftp()
sftp.unlink(path)
sftp.close()
return True
def run(self, cmd):
+ extra = {'_cmd': cmd}
+ self.logger.debug('Executing command', extra=extra)
+
# based on exec_command()
bufsize = -1
t = self.client.get_transport()
@@ -227,9 +255,15 @@ class ParamikoSSHClient(BaseSSHClient):
status = chan.recv_exit_status()
so = stdout.read()
se = stderr.read()
+
+ extra = {'_status': status, '_stdout': so, '_stderr': se}
+ self.logger.debug('Command finished', extra=extra)
+
return [so, se, status]
def close(self):
+ self.logger.debug('Closing server connection')
+
self.client.close()
return True
@@ -288,17 +322,6 @@ class ShellOutSSHClient(BaseSSHClient):
def close(self):
return True
- def _get_and_setup_logger(self):
- logger = logging.getLogger('libcloud.compute.ssh')
- path = os.getenv('LIBCLOUD_DEBUG')
-
- if path:
- handler = logging.FileHandler(path)
- logger.addHandler(handler)
- logger.setLevel(logging.DEBUG)
-
- return logger
-
def _get_base_ssh_command(self):
cmd = ['ssh']
http://git-wip-us.apache.org/repos/asf/libcloud/blob/df6dc611/libcloud/test/compute/test_ssh_client.py
----------------------------------------------------------------------
diff --git a/libcloud/test/compute/test_ssh_client.py b/libcloud/test/compute/test_ssh_client.py
index 840ff98..57df407 100644
--- a/libcloud/test/compute/test_ssh_client.py
+++ b/libcloud/test/compute/test_ssh_client.py
@@ -17,9 +17,12 @@
from __future__ import absolute_import
from __future__ import with_statement
+import os
import sys
+import tempfile
import unittest
+from libcloud import _init_once
from libcloud.compute.ssh import ParamikoSSHClient
from libcloud.compute.ssh import ShellOutSSHClient
from libcloud.compute.ssh import have_paramiko
@@ -41,6 +44,9 @@ class ParamikoSSHClientTests(unittest.TestCase):
'username': 'ubuntu',
'key': '~/.ssh/ubuntu_ssh',
'timeout': '600'}
+ _, self.tmp_file = tempfile.mkstemp()
+ os.environ['LIBCLOUD_DEBUG'] = self.tmp_file
+ _init_once()
self.ssh_cli = ParamikoSSHClient(**conn_params)
@patch('paramiko.SSHClient', Mock)
@@ -64,6 +70,7 @@ class ParamikoSSHClientTests(unittest.TestCase):
'look_for_keys': False,
'port': 22}
mock.client.connect.assert_called_once_with(**expected_conn)
+ self.assertLogMsg('Connecting to server')
@patch('paramiko.SSHClient', Mock)
def test_create_without_credentials(self):
@@ -112,9 +119,12 @@ class ParamikoSSHClientTests(unittest.TestCase):
mode='w')
mock.run(sd)
+
# Make assertions over 'run' method
mock_cli.get_transport().open_session().exec_command \
.assert_called_once_with(sd)
+ self.assertLogMsg('Executing command (cmd=/root/random_script.sh)')
+ self.assertLogMsg('Command finished')
mock.close()
@@ -131,8 +141,16 @@ class ParamikoSSHClientTests(unittest.TestCase):
mock.delete(sd)
# Make assertions over the 'delete' method
mock.client.open_sftp().unlink.assert_called_with(sd)
+ self.assertLogMsg('Deleting file')
mock.close()
+ self.assertLogMsg('Closing server connection')
+
+ def assertLogMsg(self, expected_msg):
+ with open(self.tmp_file, 'r') as fp:
+ content = fp.read()
+
+ self.assertTrue(content.find(expected_msg) != -1)
if not ParamikoSSHClient:
@@ -193,9 +211,9 @@ class ShellOutSSHClientTests(unittest.TestCase):
self.assertEqual(cmd1, ['ssh', 'root@localhost'])
self.assertEqual(cmd2, ['ssh', '-i', '/home/my.key',
- 'root@localhost'])
+ 'root@localhost'])
self.assertEqual(cmd3, ['ssh', '-i', '/home/my.key',
- '-oConnectTimeout=5', 'root@localhost'])
+ '-oConnectTimeout=5', 'root@localhost'])
if __name__ == '__main__':
http://git-wip-us.apache.org/repos/asf/libcloud/blob/df6dc611/libcloud/utils/logging.py
----------------------------------------------------------------------
diff --git a/libcloud/utils/logging.py b/libcloud/utils/logging.py
new file mode 100644
index 0000000..e95ca9c
--- /dev/null
+++ b/libcloud/utils/logging.py
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Taken from https://github.com/Kami/python-extra-log-formatters
+
+from __future__ import absolute_import
+
+import logging
+
+__all__ = [
+ 'ExtraLogFormatter'
+]
+
+
+class ExtraLogFormatter(logging.Formatter):
+ """
+ Custom log formatter which attaches all the attributes from the "extra"
+ dictionary which start with an underscore to the end of the log message.
+
+ For example:
+ extra={'_id': 'user-1', '_path': '/foo/bar'}
+ """
+ def format(self, record):
+ custom_attributes = dict([(k, v) for k, v in record.__dict__.items()
+ if k.startswith('_')])
+ custom_attributes = self._dict_to_str(custom_attributes)
+
+ msg = logging.Formatter.format(self, record)
+ msg = '%s (%s)' % (msg, custom_attributes)
+ return msg
+
+ def _dict_to_str(self, dictionary):
+ result = ['%s=%s' % (k[1:], str(v)) for k, v in dictionary.items()]
+ result = ','.join(result)
+ return result
[2/2] git commit: docs: Re-generate fixtures.
Posted by to...@apache.org.
docs: Re-generate fixtures.
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/d4ab3ab9
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/d4ab3ab9
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/d4ab3ab9
Branch: refs/heads/trunk
Commit: d4ab3ab9015aead0e0ece2a0da7666db89e7f105
Parents: df6dc61
Author: Tomaz Muraus <to...@apache.org>
Authored: Mon Oct 21 22:18:52 2013 +0200
Committer: Tomaz Muraus <to...@apache.org>
Committed: Mon Oct 21 22:18:52 2013 +0200
----------------------------------------------------------------------
docs/storage/_supported_methods_cdn.rst | 2 --
docs/storage/_supported_methods_main.rst | 2 --
docs/storage/_supported_providers.rst | 2 --
3 files changed, 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/d4ab3ab9/docs/storage/_supported_methods_cdn.rst
----------------------------------------------------------------------
diff --git a/docs/storage/_supported_methods_cdn.rst b/docs/storage/_supported_methods_cdn.rst
index 7ee989e..6955e4c 100644
--- a/docs/storage/_supported_methods_cdn.rst
+++ b/docs/storage/_supported_methods_cdn.rst
@@ -7,7 +7,6 @@ Provider enable container cdn enable object cdn get contain
`CloudFiles (UK)`_ no yes yes yes
`CloudFiles (US)`_ no yes yes yes
`Google Storage`_ no no no no
-`Local Storage`_ yes yes yes yes
`Nimbus`_ no no no no
`Ninefold`_ yes no yes no
`Amazon S3 (standard)`_ no no no no
@@ -25,7 +24,6 @@ Provider enable container cdn enable object cdn get contain
.. _`CloudFiles (US)`: http://www.rackspace.com/
.. _`Dummy Storage Provider`: http://example.com
.. _`Google Storage`: http://cloud.google.com/
-.. _`Local Storage`: http://example.com
.. _`Nimbus`: https://nimbus.io/
.. _`Ninefold`: http://ninefold.com/
.. _`Amazon S3 (standard)`: http://aws.amazon.com/s3/
http://git-wip-us.apache.org/repos/asf/libcloud/blob/d4ab3ab9/docs/storage/_supported_methods_main.rst
----------------------------------------------------------------------
diff --git a/docs/storage/_supported_methods_main.rst b/docs/storage/_supported_methods_main.rst
index 4ce814a..151f29b 100644
--- a/docs/storage/_supported_methods_main.rst
+++ b/docs/storage/_supported_methods_main.rst
@@ -7,7 +7,6 @@ Provider list containers list objects create container dele
`CloudFiles (UK)`_ yes no yes yes yes yes yes yes yes
`CloudFiles (US)`_ yes no yes yes yes yes yes yes yes
`Google Storage`_ yes no yes yes yes yes yes yes yes
-`Local Storage`_ yes no yes yes yes yes yes yes no
`Nimbus`_ no no no no no no yes no no
`Ninefold`_ yes no yes yes yes yes yes yes no
`Amazon S3 (standard)`_ yes no yes yes yes yes yes yes yes
@@ -25,7 +24,6 @@ Provider list containers list objects create container dele
.. _`CloudFiles (US)`: http://www.rackspace.com/
.. _`Dummy Storage Provider`: http://example.com
.. _`Google Storage`: http://cloud.google.com/
-.. _`Local Storage`: http://example.com
.. _`Nimbus`: https://nimbus.io/
.. _`Ninefold`: http://ninefold.com/
.. _`Amazon S3 (standard)`: http://aws.amazon.com/s3/
http://git-wip-us.apache.org/repos/asf/libcloud/blob/d4ab3ab9/docs/storage/_supported_providers.rst
----------------------------------------------------------------------
diff --git a/docs/storage/_supported_providers.rst b/docs/storage/_supported_providers.rst
index ac01c17..e8d76bb 100644
--- a/docs/storage/_supported_providers.rst
+++ b/docs/storage/_supported_providers.rst
@@ -7,7 +7,6 @@ Provider Documentation Pro
`CloudFiles (UK)`_ CLOUDFILES_UK :mod:`libcloud.storage.drivers.cloudfiles` :class:`CloudFilesUKStorageDriver`
`CloudFiles (US)`_ CLOUDFILES_US :mod:`libcloud.storage.drivers.cloudfiles` :class:`CloudFilesUSStorageDriver`
`Google Storage`_ :doc:`Click </storage/drivers/google_storage>` GOOGLE_STORAGE :mod:`libcloud.storage.drivers.google_storage` :class:`GoogleStorageDriver`
-`Local Storage`_ LOCAL :mod:`libcloud.storage.drivers.local` :class:`LocalStorageDriver`
`Nimbus`_ NIMBUS :mod:`libcloud.storage.drivers.nimbus` :class:`NimbusStorageDriver`
`Ninefold`_ NINEFOLD :mod:`libcloud.storage.drivers.ninefold` :class:`NinefoldStorageDriver`
`Amazon S3 (standard)`_ S3 :mod:`libcloud.storage.drivers.s3` :class:`S3StorageDriver`
@@ -25,7 +24,6 @@ Provider Documentation Pro
.. _`CloudFiles (US)`: http://www.rackspace.com/
.. _`Dummy Storage Provider`: http://example.com
.. _`Google Storage`: http://cloud.google.com/
-.. _`Local Storage`: http://example.com
.. _`Nimbus`: https://nimbus.io/
.. _`Ninefold`: http://ninefold.com/
.. _`Amazon S3 (standard)`: http://aws.amazon.com/s3/