# Ubuntu Testing Automation Harness
# Copyright 2012 Canonical Ltd.
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License version 3, as published
# by the Free Software Foundation.
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranties of
# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
# PURPOSE. See the GNU General Public License for more details.
# You should have received a copy of the GNU General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
"""
SSH based machine class for a provisioned system
and SSHMixin for every machine class that needs SSH support.
"""
import logging
import os
import signal
import socket
from stat import S_ISDIR
from traceback import format_exc
import paramiko
import utah.timeout
from utah.config import config
from utah.provisioning.exceptions import UTAHProvisioningException
from utah.provisioning.provisioning import Machine
from utah.retry import retry
[docs]class SSHMixin(object):
"""Provide methods for machines accessed via ssh."""
def __init__(self, *args, **kwargs):
# Note: Since this is a mixin it doesn't expect any argument
# However, it calls super to initialize any other mixins in the mro
super(SSHMixin, self).__init__(*args, **kwargs)
self.initialize()
[docs] def initialize(self):
"""SSH mixin initialization.
Use this method when it isn't appropriate to follow the MRO as in
__init__
"""
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh_client = ssh_client
self.ssh_logger = logging.getLogger('ssh')
[docs] def run(self, command, quiet=False, root=False, command_timeout=None):
"""Run a command through SSH.
:param command: Command to execute on the remote machine
:type command: str | list(str)
:param quiet:
Use debug level to log failure return command message when this
flag is set. Otherwise, use warning.
:type quiet: bool
:param root:
Run command as root user when this flag is set. Otherwise, use the
user specified in the configuration file.
:type root: bool
:param command_timeout:
Amount of time in seconds to wait for the command to complete.
.. seealso:: :func:`utah.timeout.timeout`
:type command_timeout: int
"""
if isinstance(command, basestring):
commandstring = command
else:
commandstring = ' '.join(command)
if root:
user = 'root'
else:
user = config.user
self.activecheck()
# Some commands expect run to return the output status of the command
# We're going to try the method described here:
# http://stackoverflow.com/questions/3562403/
# With additions from here:
# http://od-eon.com/blogs/
# stefan/automating-remote-commands-over-ssh-paramiko/
self.ssh_logger.debug('Connecting SSH')
retval = 'not run'
try:
self.ssh_client.connect(self.name,
username=user,
key_filename=config.sshprivatekey)
self.ssh_logger.debug('Opening SSH session')
channel = self.ssh_client.get_transport().open_session()
self.ssh_logger.info('Running command through SSH: {}'
.format(commandstring))
stdout = channel.makefile('rb')
stderr = channel.makefile_stderr('rb')
# Get remaining SIGALRM timeout
alarm_timeout = signal.alarm(0)
signal.alarm(alarm_timeout)
# Discard 0 or None timeout values
timeouts = filter(None, [command_timeout, alarm_timeout])
if timeouts:
timeout = min(timeouts)
channel.settimeout(timeout)
channel.exec_command(commandstring)
try:
# Make sure socket.timeout is raised
channel.recv(0)
except socket.timeout:
self.ssh_logger.debug(format_exc())
raise utah.timeout.UTAHTimeout(
'SSH command timed out {!r} after {} seconds'
.format(commandstring, timeout))
retval = channel.recv_exit_status()
except paramiko.BadHostKeyException as err:
self.ssh_logger.debug(format_exc())
raise UTAHProvisioningException(
'Host key exception encountered; '
'machine may be in use by another process: {}'
.format(err), external=True)
except (paramiko.AuthenticationException, paramiko.SSHException,
socket.error) as err:
self.ssh_logger.debug(format_exc())
raise UTAHProvisioningException(err)
finally:
self.ssh_logger.debug('Closing SSH connection')
self.ssh_client.close()
if retval == 0:
self.ssh_logger.debug('Return code: {}'.format(retval))
else:
if quiet:
log_method = self.ssh_logger.debug
else:
log_method = self.ssh_logger.warning
log_method(
'SSH command ({}) failed with return code: {}'
.format(commandstring, retval))
self.ssh_logger.debug('Standard output follows:')
stdout_lines = stdout.readlines()
for line in stdout_lines:
self.ssh_logger.debug(line.strip())
self.ssh_logger.debug('Standard error follows:')
stderr_lines = stderr.readlines()
for line in stderr_lines:
self.ssh_logger.debug(line.strip())
return retval, ''.join(stdout_lines), ''.join(stderr_lines)
@staticmethod
def _check_files(files):
failed = []
for f in files:
if not os.path.isfile(f):
failed.append(f)
if len(failed):
msg = 'Files do not exist: {}'.format(' '.join(failed))
err = UTAHProvisioningException(msg)
err.files = failed
raise err
[docs] def uploadfiles(self, files, target=os.path.normpath('/tmp/')):
"""Copy a file or list of files to a target directory on the machine.
:param files: File or list of files to upload
:type files: list or str
:param target: Remote path to upload files
:type target: str
"""
if isinstance(files, basestring):
files = [files]
self._check_files(files)
self.activecheck()
sftp_client = None
try:
self.ssh_client.connect(self.name,
username=config.user,
key_filename=config.sshprivatekey)
sftp_client = self.ssh_client.open_sftp()
for localpath in files:
self.ssh_logger.info(
'Uploading %s from the host to %s on the machine',
localpath,
target)
remotepath = os.path.join(target, os.path.basename(localpath))
sftp_client.put(localpath, remotepath)
except paramiko.BadHostKeyException as err:
self.ssh_logger.debug(format_exc())
raise UTAHProvisioningException(
'Host key exception encountered; '
'machine may be in use by another process: {}'
.format(err), external=True)
except (paramiko.AuthenticationException, paramiko.SSHException,
socket.error) as err:
self.ssh_logger.debug(format_exc())
raise UTAHProvisioningException(err)
finally:
self.ssh_client.close()
if sftp_client:
sftp_client.close()
[docs] def downloadfiles(self, files, target=os.path.normpath('/tmp/')):
"""Copy a file or list of files from the machine to a local target.
:param files: File or list of files to download
:type files: list or str
:param target: Local path to download files
:type target: str
"""
if isinstance(files, basestring):
files = [files]
self.activecheck()
if os.path.isdir(target):
get_localpath = lambda remotepath: \
os.path.join(target, os.path.basename(remotepath))
else:
get_localpath = lambda remotepath: target
sftp_client = None
try:
self.ssh_client.connect(self.name,
username=config.user,
key_filename=config.sshprivatekey)
sftp_client = self.ssh_client.open_sftp()
for remotepath in files:
localpath = get_localpath(remotepath)
self.ssh_logger.info('Downloading {} from the machine '
'to {} on the host'
.format(remotepath, target))
sftp_client.get(remotepath, localpath)
except paramiko.BadHostKeyException as err:
self.ssh_logger.debug(format_exc())
raise UTAHProvisioningException(
'Host key exception encountered; '
'machine may be in use by another process: {}'
.format(err), external=True)
except (paramiko.AuthenticationException, paramiko.SSHException,
socket.error) as err:
self.ssh_logger.debug(format_exc())
raise UTAHProvisioningException(err)
finally:
self.ssh_client.close()
if sftp_client:
sftp_client.close()
[docs] def downloadfilesrecursive(self, files, target=os.path.normpath('/tmp/')):
"""Recursively copy files to the target directory on the machine.
:param files: File or list of files to download
:type files: list or str
:param target: Local path to download files
:type target: str
"""
# TODO: See if we can make this the default downloadfiles
self.activecheck()
self.ssh_client.connect(self.name,
username=config.user,
key_filename=config.sshprivatekey)
sftp_client = self.ssh_client.open_sftp()
myfiles = []
if isinstance(files, basestring):
files = [files]
for myfile in files:
newtarget = os.path.join(target, os.path.basename(myfile))
if S_ISDIR(sftp_client.stat(myfile).st_mode):
self.ssh_logger.debug('%s is a directory, recursing', myfile)
if not os.path.isdir(newtarget):
self.ssh_logger.debug('Attempting to create %s', newtarget)
os.makedirs(newtarget)
myfiles = [os.path.join(myfile, x)
for x in sftp_client.listdir(myfile)]
self.downloadfilesrecursive(myfiles, newtarget)
else:
self.downloadfiles(myfile, newtarget)
[docs] def sshcheck(self):
"""Check if the machine is available via ssh.
.. seealso:: :func:`utah.retry.retry`, :meth:`sshpoll`
"""
self.ssh_logger.info('Checking for ssh availability')
try:
self.ssh_client.connect(self.name,
username=config.user,
key_filename=config.sshprivatekey)
except paramiko.BadHostKeyException as err:
self.ssh_logger.debug(format_exc())
raise UTAHProvisioningException(
'Host key exception encountered; '
'machine may be in use by another process: {}'
.format(err), external=True)
except (paramiko.AuthenticationException,
paramiko.SSHException) as err:
self.ssh_logger.debug(format_exc())
raise UTAHProvisioningException(err)
except socket.error as err:
self.ssh_logger.debug(format_exc())
raise UTAHProvisioningException(str(err), retry=True)
finally:
self.ssh_client.close()
[docs] def sshpoll(self, timeout=config.boottimeout,
checktimeout=config.checktimeout, logmethod=None):
"""Run sshcheck over and over until timeout expires.
:param timeout: Overall timeout for checking
:type timeout: int or None
:param checktimeout: Time between each check
:type checktimeout: int
:param logmethod: Method to use for logging
:type logmethod: function
"""
if logmethod is None:
logmethod = self.ssh_logger.debug
utah.timeout.timeout(timeout, retry, self.sshcheck,
logmethod=logmethod, retry_timeout=checktimeout)
[docs] def activecheck(self):
"""Start the machine if needed, and check for SSH login."""
self.logger.debug('Checking if machine is active')
self.provisioncheck()
if not self.active:
self._start()
self.sshcheck()
[docs]class ProvisionedMachine(SSHMixin, Machine):
"""A machine that is provisioned and can be accessed through ssh."""
def __init__(self, name):
SSHMixin.initialize(self)
self.name = name
self._loggersetup()
# No cleanup needed for systems that are already provisioned
self.clean = False
self.active = False
self.provisioned = True
# System is expected to be available already, so there's no need to
# wait before trying to connect through ssh
self.check_timeout = 3
self.connectivity_timeout = 60
[docs] def activecheck(self):
"""Check if machine is active.
Given that the machine is already provisioned, it's considered to be
active as long as it's reachable through ssh.
"""
if not self.active:
try:
self.pingpoll(timeout=self.connectivity_timeout,
checktimeout=self.check_timeout)
except utah.timeout.UTAHTimeout:
self.ssh_logger.debug(format_exc())
# Ignore timeout for ping, since depending on the network
# configuration ssh might still work despite the ping failure.
self.logger.warning('Network connectivity (ping) failure')
self.sshpoll(timeout=self.connectivity_timeout,
checktimeout=self.check_timeout)
self.active = True