# Copyright (C) 2011, 2012 Lars Wirzenius
# Copyright (C) 2012 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import errno
import fcntl
import logging
import os
import select
import subprocess
import time
import cliapp
def runcmd(argv, *args, **kwargs):
'''Run external command or pipeline.
Example: ``runcmd(['grep', 'foo'], ['wc', '-l'],
feed_stdin='foo\nbar\n')``
Return the standard output of the command.
Raise ``cliapp.AppException`` if external command returns
non-zero exit code. ``*args`` and ``**kwargs`` are passed
onto ``subprocess.Popen``.
'''
our_options = (
('ignore_fail', False),
('log_error', True),
)
opts = {}
for name, default in our_options:
opts[name] = default
if name in kwargs:
opts[name] = kwargs[name]
del kwargs[name]
exit_code, out, err = runcmd_unchecked(argv, *args, **kwargs)
if exit_code != 0:
msg = 'Command failed: %s\n%s\n%s' % (' '.join(argv), out, err)
if opts['ignore_fail']:
if opts['log_error']:
logging.info(msg)
else:
if opts['log_error']:
logging.error(msg)
raise cliapp.AppException(msg)
return out
def runcmd_unchecked(argv, *argvs, **kwargs):
'''Run external command or pipeline.
Return the exit code, and contents of standard output and error
of the command.
See also ``runcmd``.
'''
argvs = [argv] + list(argvs)
logging.debug('run external command: %r', argvs)
def pop_kwarg(name, default):
if name in kwargs:
value = kwargs[name]
del kwargs[name]
return value
else:
return default
def noop(_):
pass
feed_stdin = pop_kwarg('feed_stdin', '')
pipe_stdin = pop_kwarg('stdin', subprocess.PIPE)
pipe_stdout = pop_kwarg('stdout', subprocess.PIPE)
pipe_stderr = pop_kwarg('stderr', subprocess.PIPE)
stdout_callback = pop_kwarg('stdout_callback', noop)
stderr_callback = pop_kwarg('stderr_callback', noop)
output_timeout = pop_kwarg('output_timeout', None)
timeout_callback = pop_kwarg('timeout_callback', None)
try:
pipeline = _build_pipeline(argvs,
pipe_stdin,
pipe_stdout,
pipe_stderr,
kwargs)
return _run_pipeline(pipeline, feed_stdin, pipe_stdin,
pipe_stdout, pipe_stderr,
stdout_callback, stderr_callback,
output_timeout, timeout_callback)
except OSError as e: # pragma: no cover
if e.errno == errno.ENOENT and e.filename is None:
e.filename = argv[0]
raise e
else:
raise
def _build_pipeline(argvs, pipe_stdin, pipe_stdout, pipe_stderr, kwargs):
procs = []
if pipe_stderr == subprocess.PIPE:
# Make pipe for all subprocesses to share
rpipe, wpipe = os.pipe()
stderr = wpipe
else:
stderr = pipe_stderr
for i, argv in enumerate(argvs):
if i == 0 and i == len(argvs) - 1:
stdin = pipe_stdin
stdout = pipe_stdout
elif i == 0:
stdin = pipe_stdin
stdout = subprocess.PIPE
elif i == len(argvs) - 1:
stdin = procs[-1].stdout
stdout = pipe_stdout
else:
stdin = procs[-1].stdout
stdout = subprocess.PIPE
p = subprocess.Popen(argv, stdin=stdin, stdout=stdout,
stderr=stderr, close_fds=True, **kwargs)
if i != 0:
# Popen leaves this fd open in the parent,
# if we don't close it then the process at the
# write end won't receive a SIGPIPE and so won't terminate,
# this can lead to non-termination in pipelines
# that should terminate immediately, e.g. cat /dev/zero | false
stdin.close()
procs.append(p)
if pipe_stderr == subprocess.PIPE:
# Ensure only subprocesses hold the write end of the pipe, so we get
# EOF when they all terminate
os.close(wpipe)
# Allow reading of the stderr of every process as the stderr of
# the last element
procs[-1].stderr = os.fdopen(rpipe, 'rb')
return procs
def _run_pipeline(procs, feed_stdin, pipe_stdin, pipe_stdout, pipe_stderr,
stdout_callback, stderr_callback, output_timeout,
timeout_callback):
stdout_eof = False
stderr_eof = False
out = []
err = []
pos = 0
io_size = 1024
latest_output = time.time()
timeout = False
def set_nonblocking(fd):
flags = fcntl.fcntl(fd, fcntl.F_GETFL, 0)
flags = flags | os.O_NONBLOCK
fcntl.fcntl(fd, fcntl.F_SETFL, flags)
if feed_stdin and pipe_stdin == subprocess.PIPE:
set_nonblocking(procs[0].stdin.fileno())
if pipe_stdout == subprocess.PIPE:
set_nonblocking(procs[-1].stdout.fileno())
if pipe_stderr == subprocess.PIPE:
set_nonblocking(procs[-1].stderr.fileno())
def still_running():
for p in procs:
p.poll()
for p in procs:
if p.returncode is None:
return True
if pipe_stdout == subprocess.PIPE and not stdout_eof:
return True # pragma: no cover
if pipe_stderr == subprocess.PIPE and not stderr_eof:
return True # pragma: no cover
return False
while not timeout and still_running():
rlist = []
if not stdout_eof and pipe_stdout == subprocess.PIPE:
rlist.append(procs[-1].stdout)
if not stderr_eof and pipe_stderr == subprocess.PIPE:
rlist.append(procs[-1].stderr)
wlist = []
if pipe_stdin == subprocess.PIPE and pos < len(feed_stdin):
wlist.append(procs[0].stdin)
if rlist or wlist:
try:
r, w, _ = select.select(rlist, wlist, [], output_timeout)
except select.error as e: # pragma: no cover
if e.args[0] == errno.EINTR:
break
raise
else:
break # Let's not busywait waiting for processes to die.
now = time.time()
time_since_output = now - latest_output
timeout = (output_timeout is not None and
time_since_output >= output_timeout)
if procs[0].stdin in w and pos < len(feed_stdin):
data = feed_stdin[pos:pos + io_size]
procs[0].stdin.write(data)
pos += len(data)
if pos >= len(feed_stdin):
procs[0].stdin.close()
if procs[-1].stdout in r:
data = procs[-1].stdout.read(io_size)
if data:
data_new = stdout_callback(data)
if data_new is None:
data_new = data
out.append(data_new)
else:
stdout_eof = True
timeout = False
if procs[-1].stderr in r:
data = procs[-1].stderr.read(io_size)
if data:
data_new = stderr_callback(data)
if data_new is None:
data_new = data
err.append(data_new)
else:
stderr_eof = True
timeout = False
while not timeout and still_running():
for p in procs:
if p.returncode is None:
p.wait()
if timeout and timeout_callback: # pragma: no cover
timeout_callback()
errorcodes = [p.returncode for p in procs if p.returncode != 0] or [0]
# Ensure that the pipeline doesn't leak file descriptors
if procs[0].stdin is not None:
procs[0].stdin.close()
if procs[-1].stderr is not None:
procs[-1].stderr.close()
if procs[-1].stdout is not None:
procs[-1].stdout.close()
return errorcodes[-1], b''.join(out), b''.join(err)
def shell_quote(s):
'''Return a shell-quoted version of s.'''
lower_ascii = 'abcdefghijklmnopqrstuvwxyz'
upper_ascii = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
digits = '0123456789'
punctuation = '-_/=.,:'
safe = set(lower_ascii + upper_ascii + digits + punctuation)
quoted = []
for c in s:
if c in safe:
quoted.append(c)
elif c == "'":
quoted.append('"\'"')
else:
quoted.append("'%c'" % c)
return ''.join(quoted)
def ssh_runcmd(target, argv, **kwargs): # pragma: no cover
'''Run command in argv on remote host target.
This is similar to runcmd, but the command is run on the remote
machine. The command is given as an argv array; elements in the
array are automatically quoted so they get passed to the other
side correctly.
An optional ``tty=`` parameter can be passed to ``ssh_runcmd`` in
order to force or disable pseudo-tty allocation. This is often
required to run ``sudo`` on another machine and might be useful
in other situations as well. Supported values are ``tty=True`` for
forcing tty allocation, ``tty=False`` for disabling it and
``tty=None`` for not passing anything tty related to ssh.
With the ``tty`` option,
``cliapp.runcmd(['ssh', '-tt', 'user@host', '--', 'sudo', 'ls'])``
can be written as
``cliapp.ssh_runcmd('user@host', ['sudo', 'ls'], tty=True)``
which is more intuitive.
Arbitrary command line options to the ssh command can be given
with the ``ssh_options`` argument. For example:
``cliapp.ssh_runcmd(
'user@host', ['sudo', 'ls'], ssh_options=['-oStrictHostChecking=no'])``
The remote command is run in the user's home directory, by
default. The directory can be changed with the keyword argument
``remote_cwd``.
The target is given as-is to ssh, and may use any syntax ssh
accepts.
Environment variables may or may not be passed to the remote
machine: this is dependent on the ssh and sshd configurations.
Invoke env(1) explicitly to pass in the variables you need to
exist on the other end.
Pipelines are not supported.
'''
ssh_argv = ['ssh']
tty = kwargs.pop('tty', None)
if tty:
ssh_argv.append('-tt')
elif tty is False:
ssh_argv.append('-T')
more_options = kwargs.pop('ssh_options', [])
ssh_argv.extend(list(map(shell_quote, more_options)))
ssh_argv.append(target)
ssh_argv.append('--')
remote_cwd = kwargs.pop('remote_cwd', None)
if remote_cwd:
ssh_argv.extend(list(map(shell_quote, [
'sh', '-c', 'cd "$1" && shift && exec "$@"',
'-',
remote_cwd])))
local_argv = ssh_argv + list(map(shell_quote, argv))
return runcmd(local_argv, **kwargs)