valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-07 08:58:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/develop' into sam_raet_46

Browse Source
This commit is contained in:
Samuel M Smith 2014-06-20 16:03:03 -06:00
commit 4df43dc01a
14 changed files with 1493 additions and 703 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
salt/client/__init__.py
View File

@ -212,9 +212,10 @@ class LocalClient(object):
if not pub_data:
# Failed to autnenticate, this could be a bunch of things
raise EauthAuthenticationError(
'Failed to authenticate! This could be a number of issues:'
'1: Is this user permitted to execute commands?'
'2: A disk error may have occured, check disk usage and inode usage.'
'Failed to authenticate! This is most likely because this '
'user is not permitted to execute commands, but there is a '
'small possibility that a disk error ocurred (check '
'disk/inode usage).'
)
# Failed to connect to the master and send the pub

386
salt/cloud/clouds/lxc.py
View File

@ -153,6 +153,38 @@ def _salt(fun, *args, **kw):
rkwargs['timeout'] = timeout
rkwargs.setdefault('expr_form', 'list')
kwargs.setdefault('expr_form', 'list')
ping_retries = 0
# the target(s) have environ one minute to respond
# we call 60 ping request, this prevent us
# from blindly send commands to unmatched minions
ping_max_retries = 60
ping = True
# do not check ping... if we are pinguing
if fun == 'test.ping':
ping_retries = ping_max_retries + 1
# be sure that the executors are alive
while ping_retries <= ping_max_retries:
try:
if ping_retries > 0:
time.sleep(1)
pings = conn.cmd(tgt=target,
timeout=10,
fun='test.ping')
values = pings.values()
if not values:
ping = False
for v in values:
if v is not True:
ping = False
if not ping:
raise ValueError('Unreachable')
break
except Exception:
ping = False
ping_retries += 1
log.error('{0} unreachable, retrying'.format(target))
if not ping:
raise SaltCloudSystemExit('Target {0} unreachable'.format(target))
jid = conn.cmd_async(tgt=target,
fun=fun,
arg=args,
@ -363,342 +395,38 @@ def create(vm_, call=None):
'''Create an lxc Container.
This function is idempotent and will try to either provision
or finish the provision of an lxc container.
NOTE: Most of the initialisation code has been moved and merged
with the lxc runner and lxc.init functions
'''
mopts = _master_opts()
if not get_configured_provider(vm_):
return
__grains__ = _salt('grains.items')
name = vm_['name']
if 'minion' not in vm_:
vm_['minion'] = {}
minion = vm_['minion']
from_container = vm_.get('from_container', None)
image = vm_.get('image', None)
vgname = vm_.get('vgname', None)
backing = vm_.get('backing', None)
snapshot = vm_.get('snapshot', False)
prov = get_configured_provider(vm_)
if not prov:
return
profile = vm_.get('profile', None)
fstype = vm_.get('fstype', None)
dnsservers = vm_.get('dnsservers', [])
lvname = vm_.get('lvname', None)
ip = vm_.get('ip', None)
mac = vm_.get('mac', None)
netmask = vm_.get('netmask', '24')
bridge = vm_.get('bridge', 'lxcbr0')
gateway = vm_.get('gateway', 'auto')
autostart = vm_.get('autostart', True)
if autostart:
autostart = "1"
else:
autostart = "0"
size = vm_.get('size', '10G')
ssh_username = vm_.get('ssh_username', 'user')
sudo = vm_.get('sudo', True)
password = vm_.get('password', 'user')
lxc_conf_unset = vm_.get('lxc_conf_unset', [])
lxc_conf = vm_.get('lxc_conf', [])
stopped = vm_.get('stopped', False)
master = vm_.get('master', None)
script = vm_.get('script', None)
script_args = vm_.get('script_args', None)
users = vm_.get('users', None)
# some backends wont support some parameters
if backing not in ['lvm']:
lvname = vgname = None
if backing in ['dir', 'overlayfs']:
fstype = None
size = None
if backing in ['dir']:
snapshot = False
for k in ['password',
'ssh_username']:
vm_[k] = locals()[k]
if not profile:
profile = {}
salt.utils.cloud.fire_event(
'event',
'starting create',
'event', 'starting create',
'salt/cloud/{0}/creating'.format(vm_['name']),
{
'name': vm_['name'],
'profile': vm_['profile'],
'provider': vm_['provider'],
},
transport=__opts__['transport']
)
if not dnsservers:
dnsservers = ['8.8.8.8', '4.4.4.4']
changes = {}
changed = False
ret = {'name': name,
'changes': changes,
'result': True,
'comment': ''}
if not users:
users = ['root']
if (
__grains__['os'] in ['Ubuntu']
and 'ubuntu' not in users
):
users.append('ubuntu')
if ssh_username not in users:
users.append(ssh_username)
if not users:
users = []
if not lxc_conf:
lxc_conf = []
if not lxc_conf_unset:
lxc_conf_unset = []
if from_container:
method = 'clone'
else:
method = 'create'
if ip is not None:
lxc_conf.append({'lxc.network.ipv4': '{0}/{1}'.format(ip, netmask)})
if mac is not None:
lxc_conf.append({'lxc.network.hwaddr': mac})
if gateway is not None:
lxc_conf.append({'lxc.network.ipv4.gateway': gateway})
if bridge is not None:
lxc_conf.append({'lxc.network.link': bridge})
lxc_conf.append({'lxc.start.auto': autostart})
changes['100_creation'] = ''
created = False
cret = {'name': name, 'changes': {}, 'result': True, 'comment': ''}
exists = _salt('lxc.exists', name)
if exists:
cret['comment'] = 'Container already exists'
cret['result'] = True
elif method == 'clone':
oexists = _salt('lxc.exists', from_container)
if not oexists:
cret['result'] = False
cret['comment'] = (
'container could not be cloned: {0}, '
'{1} does not exist'.format(name, from_container))
else:
nret = _salt('lxc.clone',
name,
orig=from_container,
snapshot=snapshot,
size=size,
backing=backing,
profile=profile)
if nret.get('error', ''):
cret['result'] = False
cret['comment'] = '{0}\n{1}'.format(
nret['error'], 'Container cloning error')
else:
cret['result'] = (
nret['cloned']
or 'already exist' in cret.get('comment', ''))
cret['comment'] += 'Container cloned\n'
cret['changes']['status'] = 'cloned'
elif method == 'create':
nret = _salt('lxc.create',
name,
template=image,
profile=profile,
fstype=fstype,
vgname=vgname,
size=size,
lvname=lvname,
backing=backing)
if nret.get('error', ''):
cret['result'] = False
cret['comment'] = nret['error']
else:
exists = (
nret['created']
or 'already exist' in nret.get('comment', ''))
cret['comment'] += 'Container created\n'
cret['changes']['status'] = 'Created'
changes['100_creation'] = cret['comment']
ret['comment'] = changes['100_creation']
if not cret['result']:
{'name': vm_['name'], 'profile': vm_['profile'],
'provider': vm_['provider'], },
transport=__opts__['transport'])
ret = {'name': vm_['name'], 'changes': {}, 'result': True, 'comment': ''}
if 'pub_key' not in vm_ and 'priv_key' not in vm_:
log.debug('Generating minion keys for {0}'.format(vm_['name']))
vm_['priv_key'], vm_['pub_key'] = salt.utils.cloud.gen_keys(
salt.config.get_cloud_config_value(
'keysize', vm_, __opts__))
# get the minion key pair to distribute back to the container
kwarg = copy.deepcopy(vm_)
kwarg['host'] = prov['target']
cret = _runner().cmd('lxc.cloud_init', [vm_['name']], kwarg=kwarg)
ret['runner_return'] = cret
if cret['result']:
ret['result'] = False
ret['comment'] = cret['comment']
_checkpoint(ret)
if cret['changes']:
created = changed = True
# edit lxc conf if any
changes['150_conf'] = ''
cret['result'] = False
cret = _salt('lxc.update_lxc_conf',
name,
lxc_conf=lxc_conf,
lxc_conf_unset=lxc_conf_unset)
if not cret['result']:
ret['result'] = False
ret['comment'] = cret['comment']
_checkpoint(ret)
if cret['changes']:
changed = True
changes['150_conf'] = 'lxc conf ok'
# start
changes['200_start'] = 'started'
ret['comment'] = changes['200_start']
# reboot if conf has changed
cret = _salt('lxc.start', name, restart=changed)
if not cret['result']:
ret['result'] = False
changes['200_start'] = cret['comment']
ret['comment'] = changes['200_start']
_checkpoint(ret)
if cret['changes']:
changed = True
# first time provisionning only, set the default user/password
changes['250_password'] = 'Passwords in place'
ret['comment'] = changes['250_password']
gid = '/.lxc.{0}.initial_pass'.format(name, False)
lxcret = _salt('lxc.run_cmd',
name, 'test -e {0}'.format(gid),
stdout=False, stderr=False)
if lxcret:
cret = _salt('lxc.set_pass',
name,
password=password, users=users)
changes['250_password'] = 'Password updated'
if not cret['result']:
ret['result'] = False
changes['250_password'] = 'Failed to update passwords'
ret['comment'] = changes['250_password']
_checkpoint(ret)
try:
lxcret = int(
_salt('lxc.run_cmd',
name,
'sh -c \'touch "{0}"; '
'test -e "{0}";echo ${{?}}\''.format(gid)))
except ValueError:
lxcret = 1
ret['result'] = not bool(lxcret)
if not cret['result']:
changes['250_password'] = 'Failed to test password file marker'
_checkpoint(ret)
changed = True
def wait_for_ip():
'''
Wait for the IP address to become available
'''
try:
data = show_instance(vm_['name'], call='full')
except Exception:
data = {'private_ips': [], 'public_ips': []}
ips = data['private_ips'] + data['public_ips']
if ips:
if ip and ip in ips:
return ip
return ips[0]
time.sleep(1)
return False
ip = salt.utils.cloud.wait_for_fun(
wait_for_ip,
timeout=config.get_cloud_config_value(
'wait_for_fun_timeout', vm_, __opts__, default=15 * 60))
changes['300_ipattrib'] = 'Got ip {0}'.format(ip)
if not ip:
changes['300_ipattrib'] = 'Cant get ip'
ret['result'] = False
ret['comment'] = changes['300_ipattrib']
_checkpoint(ret)
# set dns servers
changes['350_dns'] = 'DNS in place'
ret['comment'] = changes['350_dns']
gid = 'lxc.{0}.initial_dns'.format(name, False)
lxcret = _salt('lxc.run_cmd',
name,
'test -e {0}'.format(gid),
stdout=False, stderr=False,)
if dnsservers and not lxcret:
cret = _salt('lxc.set_dns',
name,
dnsservers=dnsservers)
changes['350_dns'] = 'DNS updated'
ret['comment'] = changes['350_dns']
if not cret['result']:
ret['result'] = False
changes['350_dns'] = 'DNS provisionning error'
ret['comment'] = changes['350_dns']
try:
lxcret = int(
_salt('lxc.run_cmd',
name,
'sh -c \'touch "{0}"; '
'test -e "{0}";echo ${{?}}\''.format(gid)))
except ValueError:
lxcret = 1
ret['result'] = not lxcret
if not cret['result']:
changes['250_password'] = 'Failed to test DNS set marker'
_checkpoint(ret)
changed = True
_checkpoint(ret)
# provision salt on the fresh container
if 'master' in minion:
changes['400_salt'] = 'This vm is a salt minion'
def testping(*args):
ping = _salt('test.ping', **{'salt_target': vm_['name']})
time.sleep(1)
if ping:
return 'OK'
raise Exception('Unresponsive {0}'.format(vm_['name']))
# if already created, test to ping before bindly go to saltify
# we ping for 1 minute
skip = False
if not created:
ping = salt.utils.cloud.wait_for_fun(testping, timeout=10)
if ping == 'OK':
skip = True
if not skip:
minion['master_port'] = mopts.get('ret_port', '4506')
vm_['ssh_host'] = ip
vm_['sudo'] = sudo
vm_['sudo_password'] = password
svm_ = copy.deepcopy(vm_)
if 'gateway' in svm_:
del svm_['gateway']
if 'ssh_gateway' in vm_:
svm_['gateway'] = ssh_gateway_opts = {}
for k in ['ssh_gateway_key',
'ssh_gateway',
'ssh_gateway_user',
'ssh_gateway_port']:
val = vm_.get(k, None)
if val:
ssh_gateway_opts[ssh_gateway_opts.get(k, k)] = val
sret = __salt__['saltify.create'](svm_)
changes['400_salt'] = 'This vm is now a salt minion'
if 'Error' in sret:
ret['result'] = False
changes['400_salt'] = pformat(sret['Error'])
else:
changed = True
ret['comment'] = changes['400_salt']
_checkpoint(ret)
changes['401_salt'] = 'Minion is alive for salt commands'
ping = salt.utils.cloud.wait_for_fun(testping, timeout=60)
if ping != 'OK':
ret['result'] = False
changes['401_salt'] = 'Unresponsive minion!'
ret['comment'] = changes['401_salt']
_checkpoint(ret)
sret = _checkpoint(ret)
if not ret['result']:
ret['Error'] = 'Error while creating {0}'.format(vm_['name'])
if not changed and ret['result']:
ret['changes'] = {}
ret['comment'] = '\n{0}'.format(sret)
ret['Error'] = 'Error while creating {0},'.format(vm_['name'])
return ret

53
salt/daemons/masterapi.py
View File

@ -314,6 +314,41 @@ class RemoteFuncs(object):
mopts['jinja_trim_blocks'] = self.opts['jinja_trim_blocks']
return mopts
def _ext_nodes(self, load):
'''
Return the results from an external node classifier if one is
specified
'''
if 'id' not in load:
log.error('Received call for external nodes without an id')
return {}
if not salt.utils.verify.valid_id(self.opts, load['id']):
return {}
# Evaluate all configured master_tops interfaces
opts = {}
grains = {}
ret = {}
if 'opts' in load:
opts = load['opts']
if 'grains' in load['opts']:
grains = load['opts']['grains']
for fun in self.tops:
if fun not in self.opts.get('master_tops', {}):
continue
try:
ret.update(self.tops[fun](opts=opts, grains=grains))
except Exception as exc:
# If anything happens in the top generation, log it and move on
log.error(
'Top function {0} failed with error {1} for minion '
'{2}'.format(
fun, exc, load['id']
)
)
return ret
def _mine_get(self, load):
'''
Gathers the data from the specified minions' mine
@ -429,7 +464,14 @@ class RemoteFuncs(object):
if os.path.isabs(load['path']) or '../' in load['path']:
# Can overwrite master files!!
return False
if not salt.utils.verify.valid_id(self.opts, load['id']):
return False
file_recv_max_size = 1024*1024 * self.opts.get('file_recv_max_size', 100)
if 'loc' in load and load['loc'] < 0:
log.error('Invalid file pointer: load[loc] < 0')
return False
if len(load['data']) + load.get('loc', 0) > file_recv_max_size:
log.error(
'Exceeding file_recv_max_size limit: {0}'.format(
@ -560,7 +602,7 @@ class RemoteFuncs(object):
return {}
if not isinstance(self.opts['peer_run'], dict):
return {}
if any(key not in load for key in ('fun', 'arg', 'id', 'tok')):
if any(key not in load for key in ('fun', 'arg', 'id')):
return {}
perms = set()
for match in self.opts['peer_run']:
@ -573,6 +615,13 @@ class RemoteFuncs(object):
if re.match(perm, load['fun']):
good = True
if not good:
# The minion is not who it says it is!
# We don't want to listen to it!
log.warn(
'Minion id {0} is not who it says it is!'.format(
load['id']
)
)
return {}
# Prepare the runner object
opts = {'fun': load['fun'],
@ -589,7 +638,7 @@ class RemoteFuncs(object):
Request the return data from a specific jid, only allowed
if the requesting minion also initialted the execution.
'''
if any(key not in load for key in ('jid', 'id', 'tok')):
if any(key not in load for key in ('jid', 'id')):
return {}
# Check that this minion can access this data
auth_cache = os.path.join(

64
salt/master.py
View File

@ -19,13 +19,11 @@ try:
except ImportError: # This is in case windows minion is importing
pass
import resource
import subprocess
import multiprocessing
import sys
# Import third party libs
import zmq
import yaml
from M2Crypto import RSA
# Import salt libs
@ -435,25 +433,6 @@ class Publisher(multiprocessing.Process):
context.term()
class WorkerTrack(object):
def __init__(self, opts):
'''
Watches the worker procs
'''
self.opts = opts
self.counter = multiprocessing.Value('i', 0)
self.lock = multiprocessing.Lock()
def finished(self):
'''
To be called when a process finishes initializing
'''
with self.lock:
self.counter.value += 1
if int(self.opts['worker_threads']) == self.counter.value:
log.info('Master is ready to receive requests!')
class ReqServer(object):
'''
Starts up the master request server, minions send results to this
@ -476,7 +455,6 @@ class ReqServer(object):
# Prepare the AES key
self.key = key
self.crypticle = crypticle
self.tracker = WorkerTrack(opts)
def __bind(self):
'''
@ -496,14 +474,11 @@ class ReqServer(object):
self.work_procs.append(MWorker(self.opts,
self.master_key,
self.key,
self.crypticle,
tracker=self.tracker)
)
self.crypticle))
for ind, proc in enumerate(self.work_procs):
log.info('Starting Salt worker process {0}'.format(ind))
proc.start()
log.info('Successfully started Salt worker process on PID: {0}'.format(proc.pid))
self.workers.bind(self.w_uri)
@ -592,8 +567,7 @@ class MWorker(multiprocessing.Process):
opts,
mkey,
key,
crypticle,
tracker=None):
crypticle):
multiprocessing.Process.__init__(self)
self.opts = opts
self.serial = salt.payload.Serial(opts)
@ -601,7 +575,6 @@ class MWorker(multiprocessing.Process):
self.mkey = mkey
self.key = key
self.k_mtime = 0
self.tracker = tracker
def __bind(self):
'''
@ -615,7 +588,6 @@ class MWorker(multiprocessing.Process):
log.info('Worker binding to socket {0}'.format(w_uri))
try:
socket.connect(w_uri)
self.tracker.finished()
while True:
try:
package = socket.recv()
@ -876,34 +848,6 @@ class AESFuncs(object):
return {}
load.pop('tok')
ret = {}
# The old ext_nodes method is set to be deprecated in 0.10.4
# and should be removed within 3-5 releases in favor of the
# "master_tops" system
if self.opts['external_nodes']:
if not salt.utils.which(self.opts['external_nodes']):
log.error(('Specified external nodes controller {0} is not'
' available, please verify that it is installed'
'').format(self.opts['external_nodes']))
return {}
cmd = '{0} {1}'.format(self.opts['external_nodes'], load['id'])
ndata = yaml.safe_load(
subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE
).communicate()[0])
if 'environment' in ndata:
saltenv = ndata['environment']
else:
saltenv = 'base'
if 'classes' in ndata:
if isinstance(ndata['classes'], dict):
ret[saltenv] = list(ndata['classes'])
elif isinstance(ndata['classes'], list):
ret[saltenv] = ndata['classes']
else:
return ret
# Evaluate all configured master_tops interfaces
opts = {}
@ -1152,7 +1096,7 @@ class AESFuncs(object):
file_recv_max_size = 1024*1024 * self.opts.get('file_recv_max_size', 100)
if 'loc' in load and load['loc'] < 0:
log.error('Should not happen: load[loc] < 0')
log.error('Invalid file pointer: load[loc] < 0')
return False
if len(load['data']) + load.get('loc', 0) > file_recv_max_size:
@ -1505,7 +1449,7 @@ class AESFuncs(object):
load['timeout'] = int(clear_load['timeout'])
except ValueError:
msg = 'Failed to parse timeout value: {0}'.format(
clear_load['tmo'])
clear_load['timeout'])
log.warn(msg)
return {}
if 'tgt_type' in clear_load:

161
salt/modules/cmdmod.py
View File

@ -7,6 +7,7 @@ access to the master root execution access to all salt minions
'''
# Import python libs
import time
import functools
import json
import glob
@ -16,6 +17,7 @@ import shutil
import subprocess
import sys
import traceback
from salt.utils import vt
# Import salt libs
import salt.utils
@ -242,7 +244,8 @@ def _run(cmd,
timeout=None,
with_communicate=True,
reset_system_locale=True,
saltenv='base'):
saltenv='base',
use_vt=False):
'''
Do the DRY thing and only call subprocess.Popen() once
'''
@ -423,38 +426,106 @@ def _run(cmd,
.format(cwd)
)
# This is where the magic happens
try:
proc = salt.utils.timed_subprocess.TimedProc(cmd, **kwargs)
except (OSError, IOError) as exc:
raise CommandExecutionError(
'Unable to run command {0!r} with the context {1!r}, reason: {2}'
.format(cmd, kwargs, exc)
)
if not use_vt:
# This is where the magic happens
try:
proc = salt.utils.timed_subprocess.TimedProc(cmd, **kwargs)
except (OSError, IOError) as exc:
raise CommandExecutionError(
'Unable to run command {0!r} with the context {1!r}, reason: {2}'
.format(cmd, kwargs, exc)
)
try:
proc.wait(timeout)
except TimedProcTimeoutError as exc:
ret['stdout'] = str(exc)
ret['stderr'] = ''
ret['retcode'] = None
try:
proc.wait(timeout)
except TimedProcTimeoutError as exc:
ret['stdout'] = str(exc)
ret['stderr'] = ''
ret['retcode'] = None
ret['pid'] = proc.process.pid
# ok return code for timeouts?
ret['retcode'] = 1
return ret
out, err = proc.stdout, proc.stderr
if rstrip:
if out is not None:
out = out.rstrip()
if err is not None:
err = err.rstrip()
ret['pid'] = proc.process.pid
# ok return code for timeouts?
ret['retcode'] = 1
return ret
out, err = proc.stdout, proc.stderr
if rstrip:
if out is not None:
out = out.rstrip()
if err is not None:
err = err.rstrip()
ret['stdout'] = out
ret['stderr'] = err
ret['pid'] = proc.process.pid
ret['retcode'] = proc.process.returncode
ret['retcode'] = proc.process.returncode
ret['stdout'] = out
ret['stderr'] = err
else:
to = ''
if timeout:
to = ' (timeout: {0}s)'.format(timeout)
log.debug('Running {0} in VT{1}'.format(cmd, to))
stdout, stderr = '', ''
now = time.time()
if timeout:
will_timeout = now + timeout
else:
will_timeout = -1
try:
proc = vt.Terminal(cmd,
shell=True,
log_stdout=True,
log_stderr=True,
env=env,
log_stdin_level=output_loglevel,
log_stdout_level=output_loglevel,
log_stderr_level=output_loglevel,
stream_stdout=True,
stream_stderr=True)
# consume output
finished = False
ret['pid'] = proc.pid
while not finished:
try:
try:
time.sleep(0.5)
try:
cstdout, cstderr = proc.recv()
except IOError:
cstdout, cstderr = '', ''
if cstdout:
stdout += cstdout
else:
cstdout = ''
if cstderr:
stderr += cstderr
else:
cstderr = ''
if not cstdout and not cstderr and not proc.isalive():
finished = True
if timeout and (time.time() > will_timeout):
ret['stderr'] = (
'SALT: Timeout after {0}s\n{1}').format(
timeout, stderr)
ret['retcode'] = None
break
except KeyboardInterrupt:
ret['stderr'] = 'SALT: User break\n{0}'.format(stderr)
ret['retcode'] = 1
break
except vt.TerminalException as exc:
log.error(
'VT: {0}'.format(exc),
exc_info=log.isEnabledFor(logging.DEBUG))
ret = {'retcode': 1, 'pid': '2'}
break
# only set stdout on sucess as we already mangled in other
# cases
ret['stdout'] = stdout
if finished:
ret['stderr'] = stderr
ret['retcode'] = proc.exitstatus
ret['pid'] = proc.pid
finally:
proc.close(terminate=True, kill=True)
try:
__context__['retcode'] = ret['retcode']
except NameError:
@ -542,6 +613,7 @@ def run(cmd,
reset_system_locale=True,
ignore_retcode=False,
saltenv='base',
use_vt=False,
**kwargs):
'''
Execute the passed command and return the output as a string
@ -602,7 +674,8 @@ def run(cmd,
quiet=quiet,
timeout=timeout,
reset_system_locale=reset_system_locale,
saltenv=saltenv)
saltenv=saltenv,
use_vt=use_vt)
if 'pid' in ret and '__pub_jid' in kwargs:
# Stuff the child pid in the JID file
@ -651,6 +724,7 @@ def run_stdout(cmd,
reset_system_locale=True,
ignore_retcode=False,
saltenv='base',
use_vt=False,
**kwargs):
'''
Execute a command, and only return the standard out
@ -695,7 +769,8 @@ def run_stdout(cmd,
quiet=quiet,
timeout=timeout,
reset_system_locale=reset_system_locale,
saltenv=saltenv)
saltenv=saltenv,
use_vt=use_vt)
lvl = _check_loglevel(output_loglevel, quiet)
if lvl is not None:
@ -732,6 +807,7 @@ def run_stderr(cmd,
reset_system_locale=True,
ignore_retcode=False,
saltenv='base',
use_vt=False,
**kwargs):
'''
Execute a command and only return the standard error
@ -776,6 +852,7 @@ def run_stderr(cmd,
quiet=quiet,
timeout=timeout,
reset_system_locale=reset_system_locale,
use_vt=use_vt,
saltenv=saltenv)
lvl = _check_loglevel(output_loglevel, quiet)
@ -813,6 +890,7 @@ def run_all(cmd,
reset_system_locale=True,
ignore_retcode=False,
saltenv='base',
use_vt=False,
**kwargs):
'''
Execute the passed command and return a dict of return data
@ -857,7 +935,8 @@ def run_all(cmd,
quiet=quiet,
timeout=timeout,
reset_system_locale=reset_system_locale,
saltenv=saltenv)
saltenv=saltenv,
use_vt=use_vt)
lvl = _check_loglevel(output_loglevel, quiet)
if lvl is not None:
@ -893,6 +972,7 @@ def retcode(cmd,
reset_system_locale=True,
ignore_retcode=False,
saltenv='base',
use_vt=False,
**kwargs):
'''
Execute a shell command and return the command's return code.
@ -937,7 +1017,8 @@ def retcode(cmd,
quiet=quiet,
timeout=timeout,
reset_system_locale=reset_system_locale,
saltenv=saltenv)
saltenv=saltenv,
use_vt=use_vt)
lvl = _check_loglevel(output_loglevel, quiet)
if lvl is not None:
@ -968,6 +1049,7 @@ def _retcode_quiet(cmd,
reset_system_locale=True,
ignore_retcode=False,
saltenv='base',
use_vt=False,
**kwargs):
'''
Helper for running commands quietly for minion startup.
@ -988,6 +1070,7 @@ def _retcode_quiet(cmd,
reset_system_locale=reset_system_locale,
ignore_retcode=ignore_retcode,
saltenv=saltenv,
use_vt=use_vt,
**kwargs)
@ -1007,6 +1090,7 @@ def script(source,
reset_system_locale=True,
__env__=None,
saltenv='base',
use_vt=False,
**kwargs):
'''
Download a script from a remote location and execute the script locally.
@ -1091,7 +1175,8 @@ def script(source,
umask=umask,
timeout=timeout,
reset_system_locale=reset_system_locale,
saltenv=saltenv)
saltenv=saltenv,
use_vt=use_vt)
_cleanup_tempfile(path)
return ret
@ -1109,6 +1194,8 @@ def script_retcode(source,
reset_system_locale=True,
__env__=None,
saltenv='base',
output_loglevel='debug',
use_vt=False,
**kwargs):
'''
Download a script from a remote location and execute the script locally.
@ -1157,6 +1244,8 @@ def script_retcode(source,
timeout=timeout,
reset_system_locale=reset_system_locale,
saltenv=saltenv,
output_loglevel=output_loglevel,
use_vt=use_vt,
**kwargs)['retcode']

4
salt/modules/config.py
View File

@ -265,7 +265,7 @@ def dot_vals(value):
return ret
def gather_bootstrap_script():
def gather_bootstrap_script(bootstrap=None):
'''
Download the salt-bootstrap script, and return the first location
downloaded to.
@ -276,6 +276,6 @@ def gather_bootstrap_script():
salt '*' config.gather_bootstrap_script
'''
ret = salt.utils.cloud.update_bootstrap(__opts__)
ret = salt.utils.cloud.update_bootstrap(__opts__, url=bootstrap)
if 'Success' in ret and len(ret['Success']['Files updated']) > 0:
return ret['Success']['Files updated'][0]

1121
salt/modules/lxc.py
View File

File diff suppressed because it is too large Load Diff

6
salt/modules/postgres.py
View File

@ -238,7 +238,11 @@ def psql_query(query, user=None, host=None, port=None, maintenance_db=None,
password=None, runas=None):
'''
Run an SQL-Query and return the results as a list. This command
only supports SELECT statements.
only supports SELECT statements. This limitation can be worked around
with a query like this:
WITH updated AS (UPDATE pg_authid SET rolconnlimit = 2000 WHERE
rolname = 'rolename' RETURNING rolconnlimit) SELECT * FROM updated;
CLI Example:

2
salt/modules/saltutil.py
View File

@ -500,6 +500,8 @@ def clear_cache():
'''
Forcibly removes all caches on a minion.
.. versionadded:: Helium
WARNING: The safest way to clear a minion cache is by first stopping
the minion and then deleting the cache files before restarting it.

30
salt/modules/seed.py
View File

@ -25,6 +25,13 @@ __func_alias__ = {
}
def _file_or_content(file_):
if os.path.exists(file_):
with open(file_) as fic:
return fic.read()
return file_
def _mount(path, ftype):
mpt = None
if ftype == 'block':
@ -141,10 +148,17 @@ def _prep_bootstrap(mpt):
shutil.copy(bs_, os.path.join(mpt, 'tmp'))
def mkconfig(config=None, tmp=None, id_=None, approve_key=True):
def mkconfig(config=None, tmp=None, id_=None, approve_key=True,
pub_key=None, priv_key=None):
'''
Generate keys and config and put them in a tmp directory.
pub_key
absolute path or file content of an optionnal preseeded salt key
priv_key
absolute path or file content of an optionnal preseeded salt key
CLI Example:
.. code-block:: bash
@ -167,11 +181,19 @@ def mkconfig(config=None, tmp=None, id_=None, approve_key=True):
fp_.write(yaml.dump(config, default_flow_style=False))
# Generate keys for the minion
salt.crypt.gen_keys(tmp, 'minion', 2048)
pubkeyfn = os.path.join(tmp, 'minion.pub')
privkeyfn = os.path.join(tmp, 'minion.pem')
if approve_key:
preseeded = pub_key and priv_key
if preseeded:
with open(pubkeyfn, 'w') as fic:
fic.write(_file_or_content(pub_key))
with open(privkeyfn, 'w') as fic:
fic.write(_file_or_content(priv_key))
os.chmod(pubkeyfn, 0600)
os.chmod(privkeyfn, 0600)
else:
salt.crypt.gen_keys(tmp, 'minion', 2048)
if approve_key and not preseeded:
with salt.utils.fopen(pubkeyfn) as fp_:
pubkey = fp_.read()
__salt__['pillar.ext']({'virtkey': [id_, pubkey]})

2
salt/returners/couchbase_return.py
View File

@ -121,7 +121,7 @@ def _get_ttl():
'''
Return the TTL that we should store our objects with
'''
return __opts__['keep_jobs'] * 60 * 60, # keep_jobs is in hours
return __opts__['keep_jobs'] * 60 * 60 # keep_jobs is in hours
#TODO: add to returner docs-- this is a new one

227
salt/runners/lxc.py
View File

@ -7,13 +7,22 @@ Control Linux Containers via Salt
# Import python libs
from __future__ import print_function
import time
import os
import copy
import logging
# Import Salt libs
from salt.utils.odict import OrderedDict
import salt.client
import salt.output
import salt.utils.virt
import salt.utils.cloud
import salt.key
log = logging.getLogger(__name__)
# Don't shadow built-in's.
__func_alias__ = {
'list_': 'list'
@ -106,12 +115,11 @@ def find_guests(names):
return ret
def init(names,
host=None,
**kwargs):
def init(names, host=None, saltcloud_mode=False, quiet=False, **kwargs):
'''
Initialize a new container
.. code-block:: bash
salt-run lxc.init name host=minion_id [cpuset=cgroups_cpuset] \\
@ -129,6 +137,10 @@ def init(names,
host
Minion to start the container on. Required.
saltcloud_mode
init the container with the saltcloud opts format instead
See lxc.init_interface module documentation
cpuset
cgroups cpuset.
@ -152,83 +164,190 @@ def init(names,
nic_opts
Extra options for network interfaces. E.g:
{"eth0": {"mac": "aa:bb:cc:dd:ee:ff", "ipv4": "10.1.1.1", "ipv6": "2001:db8::ff00:42:8329"}}
{"eth0": {"mac": "aa:bb:cc:dd:ee:ff", "ipv4": "10.1.1.1",
"ipv6": "2001:db8::ff00:42:8329"}}
start
Start the newly created container.
seed
Seed the container with the minion config and autosign its key. Default: true
Seed the container with the minion config and autosign its key.
Default: true
install
If salt-minion is not already installed, install it. Default: true
config
Optional config parameters. By default, the id is set to the name of the
container.
Optional config parameters. By default, the id is set to
the name of the container.
'''
ret = {'comment': '', 'result': True}
if host is None:
#TODO: Support selection of host based on available memory/cpu/etc.
print('A host must be provided')
return False
names = names.split(',')
print('Searching for LXC Hosts')
# TODO: Support selection of host based on available memory/cpu/etc.
ret['comment'] = 'A host must be provided'
ret['result'] = False
return ret
if isinstance(names, basestring):
names = names.split(',')
if not isinstance(names, list):
ret['comment'] = 'Container names are not formed as a list'
ret['result'] = False
return ret
log.info('Searching for LXC Hosts')
data = __salt__['lxc.list'](host, quiet=True)
for host, containers in data.items():
for name in names:
if name in sum(containers.values(), []):
print('Container \'{0}\' already exists on host \'{1}\''.format(
name, host))
return False
log.info('Container \'{0}\' already exists'
' on host \'{1}\','
' init can be a NO-OP'.format(
name, host))
if host not in data:
print('Host \'{0}\' was not found'.format(host))
return False
kw = dict((k, v) for k, v in kwargs.items() if not k.startswith('__'))
approve_key = kw.get('approve_key', True)
if approve_key:
for name in names:
kv = salt.utils.virt.VirtKey(host, name, __opts__)
if kv.authorize():
print('Container key will be preauthorized')
else:
print('Container key preauthorization failed')
return False
ret['comment'] = 'Host \'{0}\' was not found'.format(host)
ret['result'] = False
return ret
client = salt.client.get_local_client(__opts__['conf_file'])
print('Creating container(s) \'{0}\' on host \'{1}\''.format(names, host))
kw = dict((k, v) for k, v in kwargs.items() if not k.startswith('__'))
pub_key = kw.get('pub_key', None)
priv_key = kw.get('priv_key', None)
explicit_auth = pub_key and priv_key
approve_key = kw.get('approve_key', True)
seeds = {}
if approve_key and not explicit_auth:
for name in names:
seeds[name] = kwargs.get('seed', True)
try:
ping = client.cmd(name, 'test.ping', timeout=20).get(name, None)
except (TypeError, KeyError):
ping = False
curkey = os.path.join(__opts__['pki_dir'], 'minions', name)
# be sure not to seed an alrady seeded host
if ping or os.path.exists(curkey):
seeds[name] = False
kv = salt.utils.virt.VirtKey(host, name, __opts__)
if kv.authorize():
log.info('Container key will be preauthorized')
else:
ret['comment'] = 'Container key preauthorization failed'
ret['result'] = False
return ret
log.info('Creating container(s) \'{0}\''
' on host \'{1}\''.format(names, host))
cmds = []
ret = {}
for name in names:
args = [name]
cmds.append(client.cmd_iter(host,
'lxc.init',
args,
kwarg=kwargs,
timeout=600))
ret = {}
for cmd in cmds:
sub_ret = next(cmd)
if sub_ret and host in sub_ret:
if host in ret:
ret[host].append(sub_ret[host]['ret'])
else:
ret[host] = [sub_ret[host]['ret']]
else:
ret = {}
kw = kwargs
if saltcloud_mode:
kw = copy.deepcopy(kw)
kw['name'] = name
kw = client.cmd(
host, 'lxc.cloud_init_interface', args + [kw],
expr_form='list', timeout=600).get(host, {})
name = kw.pop('name', name)
# be sure not to seed an alrady seeded host
kw['seed'] = seeds[name]
if not kw['seed']:
kw.pop('seed_cmd', '')
cmds.append(
(host,
name,
client.cmd_iter(host, 'lxc.init', args, kwarg=kw, timeout=600)))
done = ret.setdefault('done', [])
errors = ret.setdefault('errors', OrderedDict())
for host, returns in ret.items():
for j_ret in returns:
if j_ret.get('created', False) or j_ret.get('cloned', False):
print('Container \'{0}\' initialized on host \'{1}\''.format(
j_ret.get('name'), host))
for ix, acmd in enumerate(cmds):
hst, container_name, cmd = acmd
containers = ret.setdefault(hst, [])
herrs = errors.setdefault(hst, OrderedDict())
serrs = herrs.setdefault(container_name, [])
sub_ret = next(cmd)
error = None
if isinstance(sub_ret, dict) and host in sub_ret:
j_ret = sub_ret[hst]
container = j_ret.get('ret', {})
if container and isinstance(container, dict):
if not container.get('result', False):
error = container
else:
error = j_ret.get('error', 'unknown error')
print('Container \'{0}\' was not initialized: {1}'.format(j_ret.get(name), error))
return ret or None
error = 'Invalid return for {0}'.format(container_name)
else:
error = sub_ret
if not error:
error = 'unknown error (no return)'
if error:
ret['result'] = False
serrs.append(error)
else:
container['container_name'] = name
containers.append(container)
done.append(container)
# marking ping status as True only and only if we have at
# least provisionned one container
ret['ping_status'] = bool(len(done))
# for all provisionned containers, last job is to verify
# - the key status
# - we can reach them
for container in done:
# explicitly check and update
# the minion key/pair stored on the master
container_name = container['container_name']
key = os.path.join(__opts__['pki_dir'], 'minions', container_name)
if explicit_auth:
fcontent = ''
if os.path.exists(key):
with open(key) as fic:
fcontent = fic.read().strip()
if pub_key.strip() != fcontent:
with open(key, 'w') as fic:
fic.write(pub_key)
fic.flush()
mid = j_ret.get('mid', None)
if not mid:
continue
def testping(**kw):
mid_ = kw['mid']
ping = client.cmd(mid_, 'test.ping', timeout=20)
time.sleep(1)
if ping:
return 'OK'
raise Exception('Unresponsive {0}'.format(mid_))
ping = salt.utils.cloud.wait_for_fun(testping, timeout=21, mid=mid)
if ping != 'OK':
ret['ping_status'] = False
ret['result'] = False
# if no lxc detected as touched (either inited or verified
# we result to False
if not done:
ret['result'] = False
if not quiet:
salt.output.display_output(ret, '', __opts__)
return ret
def cloud_init(names, host=None, quiet=False, **kwargs):
'''
Wrapper for using lxc.init in saltcloud compatibility mode
names
Name of the containers, supports a single name or a comma delimited
list of names.
host
Minion to start the container on. Required.
saltcloud_mode
init the container with the saltcloud opts format instead
'''
return __salt__['lxc.init'](names=names, host=host,
saltcloud_mode=True, quiet=quiet, **kwargs)
def _list_iter(host=None):

57
salt/states/cmd.py
View File

@ -352,6 +352,8 @@ def wait(name,
env=(),
stateful=False,
umask=None,
output_loglevel='debug',
use_vt=False,
**kwargs):
'''
Run the given command only if the watch statement calls it
@ -412,6 +414,16 @@ def wait(name,
Only run if the file specified by ``creates`` does not exist.
.. versionadded:: Helium
output_loglevel
Control the loglevel at which the output from the command is logged.
Note that the command being run will still be logged (loglevel: DEBUG)
regardless, unless ``quiet`` is used for this value.
use_vt
Use VT utils (saltstack) to stream the command output more
interactively to the console and the logs.
This is experimental.
'''
# Ignoring our arguments is intentional.
return {'name': name,
@ -436,6 +448,8 @@ def wait_script(name,
env=None,
stateful=False,
umask=None,
use_vt=False,
output_loglevel='debug',
**kwargs):
'''
Download a script from a remote source and execute it only if a watch
@ -503,6 +517,17 @@ def wait_script(name,
stateful
The command being executed is expected to return data about executing
a state
use_vt
Use VT utils (saltstack) to stream the command output more
interactively to the console and the logs.
This is experimental.
output_loglevel
Control the loglevel at which the output from the command is logged.
Note that the command being run will still be logged (loglevel: DEBUG)
regardless, unless ``quiet`` is used for this value.
'''
# Ignoring our arguments is intentional.
return {'name': name,
@ -522,9 +547,10 @@ def run(name,
env=None,
stateful=False,
umask=None,
output_loglevel='info',
output_loglevel='debug',
quiet=False,
timeout=None,
use_vt=False,
**kwargs):
'''
Run a command if certain circumstances are met. Use ``cmd.wait`` if you
@ -584,7 +610,7 @@ def run(name,
output_loglevel
Control the loglevel at which the output from the command is logged.
Note that the command being run will still be logged at loglevel INFO
Note that the command being run will still be logged (loglevel: DEBUG)
regardless, unless ``quiet`` is used for this value.
quiet
@ -602,6 +628,11 @@ def run(name,
.. versionadded:: Helium
use_vt
Use VT utils (saltstack) to stream the command output more
interactively to the console and the logs.
This is experimental.
.. note::
cmd.run supports the usage of ``reload_modules``. This functionality
@ -651,6 +682,7 @@ def run(name,
cmd_kwargs = {'cwd': cwd,
'runas': user,
'use_vt': use_vt,
'shell': shell or __grains__['shell'],
'env': env,
'umask': umask,
@ -700,6 +732,8 @@ def script(name,
stateful=False,
umask=None,
timeout=None,
use_vt=False,
output_loglevel='debug',
**kwargs):
'''
Download a script and execute it with specified arguments.
@ -780,6 +814,17 @@ def script(name,
Only run if the file specified by ``creates`` does not exist.
.. versionadded:: Helium
use_vt
Use VT utils (saltstack) to stream the command output more
interactively to the console and the logs.
This is experimental.
output_loglevel
Control the loglevel at which the output from the command is logged.
Note that the command being run will still be logged (loglevel: DEBUG)
regardless, unless ``quiet`` is used for this value.
'''
ret = {'name': name,
'changes': {},
@ -815,6 +860,8 @@ def script(name,
'template': template,
'umask': umask,
'timeout': timeout,
'output_loglevel': output_loglevel,
'use_vt': use_vt,
'saltenv': __env__})
run_check_cmd_kwargs = {
@ -876,6 +923,8 @@ def call(name,
onlyif=None,
unless=None,
creates=None,
output_loglevel='debug',
use_vt=False,
**kwargs):
'''
Invoke a pre-defined Python function with arguments specified in the state
@ -918,6 +967,8 @@ def call(name,
'runas': kwargs.get('user'),
'shell': kwargs.get('shell') or __grains__['shell'],
'env': kwargs.get('env'),
'use_vt': use_vt,
'output_loglevel': output_loglevel,
'umask': kwargs.get('umask')}
if HAS_GRP:
pgid = os.getegid()
@ -952,6 +1003,8 @@ def wait_call(name,
unless=None,
creates=None,
stateful=False,
use_vt=False,
output_loglevel='debug',
**kwargs):
# Ignoring our arguments is intentional.
return {'name': name,

76
salt/utils/cloud.py
View File

@ -8,6 +8,7 @@ import os
import sys
import codecs
import shutil
import hashlib
import socket
import tempfile
import time
@ -1948,28 +1949,61 @@ def delete_minion_cachedir(minion_id, provider, opts, base=None):
os.remove(path)
def update_bootstrap(config):
def update_bootstrap(config, url=None):
'''
Update the salt-bootstrap script
url can be either:
- The URL to fetch the bootstrap script from
- The absolute path to the bootstrap
- The content of the bootstrap script
'''
log.debug('Updating the bootstrap-salt.sh script to latest stable')
try:
import requests
except ImportError:
return {'error': (
'Updating the bootstrap-salt.sh script requires the '
'Python requests library to be installed'
)}
url = 'https://bootstrap.saltstack.com'
req = requests.get(url)
if req.status_code != 200:
return {'error': (
'Failed to download the latest stable version of the '
'bootstrap-salt.sh script from {0}. HTTP error: '
'{1}'.format(
url, req.status_code
)
)}
default_url = config.get('bootstrap_script__url',
'https://bootstrap.saltstack.com')
if not url:
url = default_url
if not url:
raise ValueError('Cant get any source to update')
if (url.startswith('http')) or ('://' in url):
log.debug('Updating the bootstrap-salt.sh script to latest stable')
try:
import requests
except ImportError:
return {'error': (
'Updating the bootstrap-salt.sh script requires the '
'Python requests library to be installed'
)}
req = requests.get(url)
if req.status_code != 200:
return {'error': (
'Failed to download the latest stable version of the '
'bootstrap-salt.sh script from {0}. HTTP error: '
'{1}'.format(
url, req.status_code
)
)}
script_content = req.text
if url == default_url:
script_name = 'bootstrap-salt.sh'
else:
script_name = os.path.basename(url)
elif os.path.exists(url):
with open(url) as fic:
script_content = fic.read()
script_name = os.path.basename(url)
# in last case, assuming we got a script content
else:
script_content = url
script_name = '{0}.sh'.format(
hashlib.sha1(script_content).hexdigest()
)
if not script_content:
raise ValueError('No content in bootstrap script !')
# Get the path to the built-in deploy scripts directory
builtin_deploy_dir = os.path.join(
@ -2044,11 +2078,11 @@ def update_bootstrap(config):
)
continue
deploy_path = os.path.join(entry, 'bootstrap-salt.sh')
deploy_path = os.path.join(entry, script_name)
try:
finished_full.append(deploy_path)
with salt.utils.fopen(deploy_path, 'w') as fp_:
fp_.write(req.text)
fp_.write(script_content)
except (OSError, IOError) as err:
log.debug(
'Failed to write the updated script: {0}'.format(err)