valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-08 09:23:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/develop' into sam_raet_14

Browse Source
This commit is contained in:
Samuel M Smith 2014-03-03 12:47:49 -07:00
commit efdce44252
15 changed files with 398 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
doc/topics/tutorials/gitfs.rst
View File

@ -228,18 +228,26 @@ following to your master config file:
.. code-block:: yaml
ext_pillar:
- git: <branch> <repo>
- git: <branch> <repo> [root=<gitroot>]
The ``<branch>`` param is the branch containing the pillar SLS tree, and the
``<repo>`` param is the URI for the repository. The below example would add the
``master`` branch of the specified repo as an external pillar source.
``master`` branch of the specified repo as an external pillar source:
.. code-block:: yaml
ext_pillar:
- git: master https://domain.com/pillar.git
Use the ``root`` parameter to use pillars from a subdirectory of the GIT
repository:
.. code-block:: yaml
ext_pillar:
- git: master https://domain.com/pillar.git root=subdirectory
More information on the git external pillar can be found :mod:`here
<salt.pillar.git_pillar>`.

41
salt/client/ssh/__init__.py
View File

@ -49,6 +49,25 @@ RSTR = '_edbc7885e4f9aac9b83b35999b68d015148caf467b78fa39c05f669c0ff89878'
# - First format pass inserts salt version and delimiter
# - Second pass at run-time and inserts optional "sudo" and command
SSH_SHIM = '''/bin/sh << 'EOF'
which ls > /dev/null 2> /dev/null
wret=$?
if [ $wret -ne 0 ]
then
if [ $wret -eq 127 ]
then
echo "The following required Packages are missing: which" >&2
else
echo 'which command error' >&2
fi
exit $wret
fi
MISS_PKG=''
if [ ! $(which tar 2>/dev/null) ]
then
MISS_PKG="$MISS_PKG tar"
fi
for py_candidate in \\
python27 \\
python2.7 \\
@ -63,6 +82,12 @@ SSH_SHIM = '''/bin/sh << 'EOF'
break
fi
done
if [ "$PYTHON" == "" ]
then
MISS_PKG="$MISS_PKG python"
fi
SALT=/tmp/.salt/salt-call
if [ {{2}} = 'md5' ]
then
@ -77,7 +102,21 @@ SSH_SHIM = '''/bin/sh << 'EOF'
fi
done
else
SUMCHECK={{2}}
if [ $(which {{2}} 2>/dev/null) ]
then
SUMCHECK={{2}}
fi
fi
if [ "$SUMCHECK" == "" ]
then
MISS_PKG="$MISS_PKG md5 or md5sum"
fi
if [ "$MISS_PKG" != "" ]
then
echo "The following required Packages are missing: $MISS_PKG" >&2
exit 127
fi
if [ $SUMCHECK = '/sbin/md5' ]

3
salt/client/ssh/shell.py
View File

@ -77,6 +77,9 @@ class Shell(object):
options.append('ConnectTimeout={0}'.format(self.timeout))
if self.opts.get('ignore_host_keys'):
options.append('StrictHostKeyChecking=no')
known_hosts = self.opts.get('known_hosts_file')
if known_hosts:
options.append('UserKnownHostsFile={0}'.format(known_hosts))
if self.port:
options.append('Port={0}'.format(self.port))
if self.priv:

4
salt/cloud/clouds/ec2.py
View File

@ -2680,6 +2680,7 @@ def create_snapshot(kwargs=None, call=None):
data = query(params, return_root=True)
return data
def delete_snapshot(kwargs=None, call=None):
'''
Delete a snapshot
@ -2704,6 +2705,7 @@ def delete_snapshot(kwargs=None, call=None):
data = query(params, return_root=True)
return data
def copy_snapshot(kwargs=None, call=None):
'''
Copy a snapshot
@ -2741,6 +2743,7 @@ def copy_snapshot(kwargs=None, call=None):
data = query(params, return_root=True)
return data
def describe_snapshot(kwargs=None, call=None):
'''
Describe a snapshot
@ -2765,4 +2768,3 @@ def describe_snapshot(kwargs=None, call=None):
data = query(params, return_root=True)
return data

17
salt/fileclient.py
View File

@ -522,6 +522,23 @@ class Client(object):
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
os.makedirs(destdir)
if url_data.scheme == 's3':
try:
salt.utils.s3.query(method='GET',
bucket=url_data.netloc,
path=url_data.path[1:],
return_bin=False,
local_file=dest,
action=None,
key=self.opts.get('s3.key', None),
keyid=self.opts.get('s3.keyid', None),
service_url=self.opts.get('s3.service_url',
None))
return dest
except Exception as ex:
raise MinionError('Could not fetch from {0}'.format(url))
if url_data.username is not None \
and url_data.scheme in ('http', 'https'):
_, netloc = url_data.netloc.split('@', 1)

8
salt/modules/tomcat.py
View File

@ -87,13 +87,13 @@ def _auth(uri):
If user & pass are missing return False
'''
try:
user = __grains__['tomcat-manager.user']
password = __grains__['tomcat-manager.passwd']
user = __grains__['tomcat-manager']['user']
password = __grains__['tomcat-manager']['passwd']
except KeyError:
try:
user = salt.utils.option('tomcat-manager.user', '', __opts__,
user = salt.utils.option('tomcat-manager:user', '', __opts__,
__pillar__)
password = salt.utils.option('tomcat-manager.passwd', '', __opts__,
password = salt.utils.option('tomcat-manager:passwd', '', __opts__,
__pillar__)
except Exception:
return False

49
salt/pillar/git_pillar.py
View File

@ -8,7 +8,10 @@ so:
.. code-block:: yaml
ext_pillar:
- git: master git://gitserver/git-pillar.git
- git: master git://gitserver/git-pillar.git root=subdirectory
The `root=` parameter is optional and used to set the subdirectory from where
to look for Pillar files (such as ``top.sls``).
Note that this is not the same thing as configuring pillar data using the
:conf_master:`pillar_roots` parameter. The branch referenced in the
@ -96,9 +99,9 @@ class GitPillar(object):
self.working_dir = ''
self.repo = None
needle = '{0} {1}'.format(self.branch, self.rp_location)
for idx, opts_dict in enumerate(self.opts['ext_pillar']):
if opts_dict.get('git', '') == '{0} {1}'.format(self.branch,
self.rp_location):
if opts_dict.get('git', '').startswith(needle):
rp_ = os.path.join(self.opts['cachedir'],
'pillar_gitfs', str(idx))
@ -203,28 +206,60 @@ def envs(branch, repo_location):
return gitpil.envs()
def _extract_key_val(kv, delim='='):
'''Extract key and value from key=val string.
Example:
>>> _extract_key_val('foo=bar')
('foo', 'bar')
'''
delim = '='
pieces = kv.split(delim)
key = pieces[0]
val = delim.join(pieces[1:])
return key, val
def ext_pillar(minion_id, pillar, repo_string):
'''
Execute a command and read the output as YAML
'''
# split the branch and repo name
branch, repo_location = repo_string.strip().split()
# split the branch, repo name and optional extra (key=val) parameters.
options = repo_string.strip().split()
branch = options[0]
repo_location = options[1]
root = ''
for extraopt in options[2:]:
# Support multiple key=val attributes as custom parameters.
DELIM = '='
if DELIM not in extraopt:
log.error(("Incorrectly formatted extra parameter."
" Missing '%s': %s"), DELIM, extraopt)
key, val = _extract_key_val(extraopt, DELIM)
if key == 'root':
root = val
else:
log.warning("Unrecognized extra parameter: %s", key)
gitpil = GitPillar(branch, repo_location, __opts__)
# environment is "different" from the branch
branch = (branch == 'master' and 'base' or branch)
# normpath is needed to remove appended '/' if root is empty string.
pillar_dir = os.path.normpath(os.path.join(gitpil.working_dir, root))
# Don't recurse forever-- the Pillar object will re-call the ext_pillar
# function
if __opts__['pillar_roots'].get(branch, []) == [gitpil.working_dir]:
if __opts__['pillar_roots'].get(branch, []) == [pillar_dir]:
return {}
gitpil.update()
opts = deepcopy(__opts__)
opts['pillar_roots'][branch] = [gitpil.working_dir]
opts['pillar_roots'][branch] = [pillar_dir]
pil = Pillar(opts, __grains__, minion_id, branch)

5
salt/runners/jobs.py
View File

@ -53,7 +53,7 @@ def active():
return ret
def lookup_jid(jid, ext_source=None):
def lookup_jid(jid, ext_source=None, output=True):
'''
Return the printout from a previously executed job
@ -84,7 +84,8 @@ def lookup_jid(jid, ext_source=None):
for mid, data in client.get_full_returns(jid, [], 0).items():
ret[mid] = data.get('ret')
salt.output.display_output(
if output:
salt.output.display_output(
{mid: ret[mid]},
data.get('out', None),
__opts__)

10
salt/state.py
View File

@ -19,7 +19,6 @@ import copy
import site
import fnmatch
import logging
import collections
import traceback
import datetime
@ -36,6 +35,7 @@ from salt._compat import string_types
from salt.utils.immutabletypes import ImmutableLazyProxy
from salt.template import compile_template, compile_template_str
from salt.exceptions import SaltRenderError, SaltReqTimeoutError, SaltException
from salt.utils.odict import OrderedDict, DefaultOrderedDict
log = logging.getLogger(__name__)
@ -1881,9 +1881,9 @@ class BaseHighState(object):
'''
Gather the top files
'''
tops = collections.defaultdict(list)
include = collections.defaultdict(list)
done = collections.defaultdict(list)
tops = DefaultOrderedDict(list)
include = DefaultOrderedDict(list)
done = DefaultOrderedDict(list)
# Gather initial top files
if self.opts['environment']:
tops[self.opts['environment']] = [
@ -1951,7 +1951,7 @@ class BaseHighState(object):
'''
Cleanly merge the top files
'''
top = collections.defaultdict(dict)
top = DefaultOrderedDict(OrderedDict)
for ctops in tops.values():
for ctop in ctops:
for saltenv, targets in ctop.items():

3
salt/states/file.py
View File

@ -951,6 +951,9 @@ def managed(name,
source
The source file to download to the minion, this source file can be
hosted on either the salt master server, or on an HTTP or FTP server.
Both HTTPS and HTTP are supported as well as downloading directly
from Amazon S3 compatible URLs with both pre-configured and automatic
IAM credentials. (see s3.get state documentation)
For files hosted on the salt file server, if the file is located on
the master in the directory named spam, and is called eggs, the source
string is salt://spam/eggs. If source is left blank or None

178
salt/states/mysql_query.py Normal file
View File

@ -0,0 +1,178 @@
# -*- coding: utf-8 -*-
'''
Execution of MySQL queries
==========================
:depends: - MySQLdb Python module
:configuration: See :py:mod:`salt.modules.mysql` for setup instructions.
The mysql_query module is used to execute queries on MySQL databases.
Its output may be stored in a file or in a grain.
.. code-block:: yaml
query_id:
mysql_query.run
- database: my_database
- query: "SELECT * FROM table;"
- output: "/tmp/query_id.txt"
'''
import sys
import os.path
def __virtual__():
'''
Only load if the mysql module is available in __salt__
'''
return 'mysql_query' if 'mysql.query' in __salt__ else False
def _get_mysql_error():
'''
Look in module context for a MySQL error. Eventually we should make a less
ugly way of doing this.
'''
return sys.modules[
__salt__['test.ping'].__module__
].__context__.pop('mysql.error', None)
def run(name,
database,
query,
output=None,
grain=None,
key=None,
overwrite=True,
**connection_args):
'''
Execute an arbitrary query on the specified database
name
Used only as an ID
database
The name of the database to execute the query on
query
The query to execute
output
grain: output in a grain
other: the file to store results
None: output to the result comment (default)
grain:
grain to store the output (need output=grain)
key:
the specified grain will be treated as a dictionnary, the result
of this state will be stored under the specified key.
overwrite:
The file or grain will be overwritten if it already exists (default)
'''
ret = {'name': name,
'changes': {},
'result': True,
'comment': 'Database {0} is already present'.format(name)}
# check if database exists
if not __salt__['mysql.db_exists'](database, **connection_args):
err = _get_mysql_error()
if err is not None:
ret['comment'] = err
ret['result'] = False
return ret
ret['result'] = None
ret['comment'] = ('Database {0} is not present'
).format(name)
return ret
# Check if execution needed
if output == 'grain':
if grain is not None and key is None:
if not overwrite and grain in __salt__['grains.ls']():
ret['comment'] = 'No execution needed. Grain ' + grain\
+ ' already set'
return ret
elif __opts__['test']:
ret['result'] = None
ret['comment'] = 'Query would execute, storing result in '\
+ 'grain: ' + grain
return ret
elif grain is not None:
if grain in __salt__['grains.ls']():
grain_value = __salt__['grains.get'](grain)
else:
grain_value = {}
if not overwrite and key in grain_value:
ret['comment'] = 'No execution needed. Grain ' + grain\
+ ':' + key + ' already set'
return ret
elif __opts__['test']:
ret['result'] = None
ret['comment'] = 'Query would execute, storing result in '\
+ 'grain: ' + grain + ':' + key
return ret
else:
ret['result'] = False
ret['comment'] = "Error: output type 'grain' needs the grain "\
+ "parameter\n"
return ret
elif output is not None:
if not overwrite and os.path.isfile(output):
ret['comment'] = 'No execution needed. File ' + output\
+ ' already set'
return ret
elif __opts__['test']:
ret['result'] = None
ret['comment'] = 'Query would execute, storing result in '\
+ 'file: ' + output
return ret
elif __opts__['test']:
ret['result'] = None
ret['comment'] = 'Query would execute, not storing result'
# The database is present, execute the query
query_result = __salt__['mysql.query'](database, query, **connection_args)
mapped_results = []
if 'results' in query_result:
for res in query_result['results']:
mapped_line = {}
for idx, col in enumerate(query_result['columns']):
mapped_line[col] = res[idx]
mapped_results.append(mapped_line)
query_result['results'] = mapped_results
ret['comment'] = query_result
if output == 'grain':
if grain is not None and key is None:
__salt__['grains.setval'](grain, query_result)
ret['changes']['query'] = "Executed. Output into grain: "\
+ grain
elif grain is not None:
if grain in __salt__['grains.ls']():
grain_value = __salt__['grains.get'](grain)
else:
grain_value = {}
grain_value[key] = query_result
__salt__['grains.setval'](grain, grain_value)
ret['changes']['query'] = "Executed. Output into grain: "\
+ grain + ":" + key
elif output is not None:
ret['changes']['query'] = "Executed. Output into " + output
with open(output, 'w') as output_file:
if 'results' in query_result:
for res in query_result['results']:
for col, val in res:
output_file.write(col + ':' + val + '\n')
else:
output_file.write(str(query_result))
else:
ret['changes']['query'] = "Executed"
return ret

15
salt/utils/__init__.py
View File

@ -1463,12 +1463,15 @@ def option(value, default='', opts=None, pillar=None):
opts = {}
if pillar is None:
pillar = {}
if value in opts:
return opts[value]
if value in pillar.get('master', {}):
return pillar['master'][value]
if value in pillar:
return pillar[value]
sources = (
(opts, value),
(pillar, 'master:{0}'.format(value)),
(pillar, value),
)
for source, val in sources:
out = traverse_dict(source, val, default)
if out is not default:
return out
return default

52
salt/utils/odict.py
View File

@ -14,7 +14,14 @@
provides an ``OrderedDict`` implementation based on::
http://code.activestate.com/recipes/576669/
It also implements a DefaultOrderedDict Class that serves as a
combination of ``OrderedDict`` and ``defaultdict``
It's source was submitted here::
http://stackoverflow.com/questions/6190331/
'''
from collections import Callable
try:
from collections import OrderedDict # pylint: disable=E0611
@ -282,3 +289,48 @@ except ImportError:
# "od.viewitems() -> a set-like object providing a view on od's items"
# return ItemsView(self)
# ## end of http://code.activestate.com/recipes/576693/ }}}
finally:
class DefaultOrderedDict(OrderedDict):
'''
Dictionary that remembers insertion order
'''
def __init__(self, default_factory=None, *a, **kw):
if (default_factory is not None and
not isinstance(default_factory, Callable)):
raise TypeError('first argument must be callable')
OrderedDict.__init__(self, *a, **kw)
self.default_factory = default_factory
def __getitem__(self, key):
try:
return OrderedDict.__getitem__(self, key)
except KeyError:
return self.__missing__(key)
def __missing__(self, key):
if self.default_factory is None:
raise KeyError(key)
self[key] = value = self.default_factory()
return value
def __reduce__(self):
if self.default_factory is None:
args = tuple()
else:
args = self.default_factory,
return type(self), args, None, None, self.items()
def copy(self):
return self.__copy__()
def __copy__(self):
return type(self)(self.default_factory, self)
def __deepcopy__(self, memo):
import copy
return type(self)(self.default_factory,
copy.deepcopy(self.items()))
def __repr__(self):
return 'DefaultOrderedDict(%s, %s)' % (self.default_factory,
OrderedDict.__repr__(self))

21
salt/utils/parsers.py
View File

@ -29,11 +29,29 @@ import salt.version as version
import salt.syspaths as syspaths
import salt.log.setup as log
from salt.utils.validate.path import is_writeable
from salt._compat import string_types
if not utils.is_windows():
import salt.cloud.exceptions
def parse_args_kwargs(args):
'''
Parse out the args and kwargs from an args string
'''
_args = []
_kwargs = {}
for arg in args:
if isinstance(arg, string_types):
arg_name, arg_value = salt.utils.parse_kwarg(arg)
if arg_name:
_kwargs[arg_name] = arg_value
else:
_args.append(arg)
return _args, _kwargs
def _sorted(mixins_or_funcs):
return sorted(
mixins_or_funcs, key=lambda mf: getattr(mf, '_mixin_prio_', 1000)
@ -1665,6 +1683,9 @@ class SaltCMDOptionParser(OptionParser, ConfigDirMixIn, MergeConfigMixIn,
else:
self.config['fun'] = self.args[1]
self.config['arg'] = self.args[2:]
# parse the args and kwargs before sending to the publish interface
self.config['arg'] = salt.client.condition_kwarg(*parse_args_kwargs(self.config['arg']))
except IndexError:
self.exit(42, '\nIncomplete options passed.\n\n')

8
tests/unit/utils/utils_test.py
View File

@ -252,6 +252,14 @@ class UtilsTestCase(TestCase):
self.assertTrue(utils.test_mode(Test=True))
self.assertTrue(utils.test_mode(tEsT=True))
def test_option(self):
test_two_level_dict = {'foo': {'bar': 'baz'}}
self.assertDictEqual({'not_found': 'nope'}, utils.option('foo:bar', {'not_found': 'nope'}))
self.assertEqual('baz', utils.option('foo:bar', {'not_found': 'nope'}, opts=test_two_level_dict))
self.assertEqual('baz', utils.option('foo:bar', {'not_found': 'nope'}, pillar={'master': test_two_level_dict}))
self.assertEqual('baz', utils.option('foo:bar', {'not_found': 'nope'}, pillar=test_two_level_dict))
def test_parse_docstring(self):
test_keystone_str = '''Management of Keystone users
============================