From 12811bfac2c76b2a7239f9d7e8cb107291fd6cd8 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 12 Jul 2015 12:44:09 +0100 Subject: [PATCH 01/26] Improve exception source code readability --- salt/client/ssh/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py index 930de1d1a4..562995da5b 100644 --- a/salt/client/ssh/__init__.py +++ b/salt/client/ssh/__init__.py @@ -200,7 +200,13 @@ class SSH(object): try: salt.client.ssh.shell.gen_key(priv) except OSError: - raise salt.exceptions.SaltClientError('salt-ssh could not be run because it could not generate keys.\n\nYou can probably resolve this by executing this script with increased permissions via sudo or by running as root.\nYou could also use the \'-c\' option to supply a configuration directory that you have permissions to read and write to.') + raise salt.exceptions.SaltClientError( + 'salt-ssh could not be run because it could not generate keys.\n\n' + 'You can probably resolve this by executing this script with ' + 'increased permissions via sudo or by running as root.\n' + 'You could also use the \'-c\' option to supply a configuration ' + 'directory that you have permissions to read and write to.' + ) self.defaults = { 'user': self.opts.get( 'ssh_user', From f375b4a8008a838c559bad99281c547e31e38c42 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 12 Jul 2015 14:16:00 +0100 Subject: [PATCH 02/26] Version string comparison is not supported under PY3 --- salt/client/ssh/shell.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/client/ssh/shell.py b/salt/client/ssh/shell.py index 4fb7806e1c..1babbbe37c 100644 --- a/salt/client/ssh/shell.py +++ b/salt/client/ssh/shell.py @@ -96,7 +96,7 @@ class Shell(object): options.append('PasswordAuthentication=yes') else: options.append('PasswordAuthentication=no') - if self.opts.get('_ssh_version', '') > '4.9': + if self.opts.get('_ssh_version', (0,)) > (4, 9): options.append('GSSAPIAuthentication=no') options.append('ConnectTimeout={0}'.format(self.timeout)) if self.opts.get('ignore_host_keys'): @@ -131,7 +131,7 @@ class Shell(object): options = ['ControlMaster=auto', 'StrictHostKeyChecking=no', ] - if self.opts['_ssh_version'] > '4.9': + if self.opts['_ssh_version'] > (4, 9): options.append('GSSAPIAuthentication=no') options.append('ConnectTimeout={0}'.format(self.timeout)) if self.opts.get('ignore_host_keys'): From cb879c7233a2164f489d564d97955c1548ce43fe Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 12 Jul 2015 14:18:43 +0100 Subject: [PATCH 03/26] Version string comparison is not supported under PY3 --- salt/client/ssh/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py index 562995da5b..da022d0ea9 100644 --- a/salt/client/ssh/__init__.py +++ b/salt/client/ssh/__init__.py @@ -1239,9 +1239,16 @@ def ssh_version(): stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() try: - return ret[1].split(b',')[0].split(b'_')[1] + version_parts = ret[1].split(b',')[0].split(b'_')[1] + parts = [] + for part in version_parts: + try: + parts.append(int(part)) + except ValueError: + return tuple(parts) + return tuple(parts) except IndexError: - return '2.0' + return (2, 0) def _convert_args(args): From 1049f4bd215f6563e7790863e5e81ad27f9bcb2a Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 12 Jul 2015 19:06:34 +0100 Subject: [PATCH 04/26] Python 3 compatibility --- salt/returners/local_cache.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/salt/returners/local_cache.py b/salt/returners/local_cache.py index 02f12101c3..13dc091f2d 100644 --- a/salt/returners/local_cache.py +++ b/salt/returners/local_cache.py @@ -19,6 +19,10 @@ import salt.payload import salt.utils import salt.utils.jid +# Import 3rd-party libs +import salt.ext.six as six + + log = logging.getLogger(__name__) # load is the published job @@ -45,8 +49,10 @@ def _jid_dir(jid): ''' Return the jid_dir for the given job id ''' - jid = str(jid) - jhash = getattr(hashlib, __opts__['hash_type'])(jid).hexdigest() + if six.PY3: + jhash = getattr(hashlib, __opts__['hash_type'])(jid.encode('utf-8')).hexdigest() + else: + jhash = getattr(hashlib, __opts__['hash_type'])(str(jid)).hexdigest() return os.path.join(_job_dir(), jhash[:2], jhash[2:]) @@ -96,7 +102,10 @@ def prep_jid(nocache=False, passed_jid=None): return prep_jid(nocache=nocache) with salt.utils.fopen(os.path.join(jid_dir_, 'jid'), 'wb+') as fn_: - fn_.write(jid) + if six.PY2: + fn_.write(jid) + else: + fn_.write(bytes(jid, 'utf-8')) if nocache: with salt.utils.fopen(os.path.join(jid_dir_, 'nocache'), 'wb+') as fn_: fn_.write('') From fc9267afa3a7ecaae3ef446575072e0e5d51d8b7 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 12 Jul 2015 19:11:10 +0100 Subject: [PATCH 05/26] Allow salt-ssh to run in both python 2 and 3 --- salt/client/ssh/__init__.py | 18 +++- salt/runners/thin.py | 10 +- salt/utils/parsers.py | 10 ++ salt/utils/thin.py | 187 ++++++++++++++++++++++++------------ 4 files changed, 156 insertions(+), 69 deletions(-) diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py index da022d0ea9..a890697e64 100644 --- a/salt/client/ssh/__init__.py +++ b/salt/client/ssh/__init__.py @@ -4,6 +4,7 @@ Create ssh executor system ''' # Import python libs from __future__ import absolute_import, print_function +import base64 import copy import getpass import json @@ -126,9 +127,9 @@ if [ -n "{{SUDO}}" ] then SUDO="sudo " fi EX_PYTHON_INVALID={EX_THIN_PYTHON_INVALID} -PYTHON_CMDS="python27 python2.7 python26 python2.6 python2 python" +PYTHON_CMDS="python3 python27 python2.7 python26 python2.6 python2 python" for py_cmd in $PYTHON_CMDS -do if "$py_cmd" -c "import sys; sys.exit(not (sys.hexversion >= 0x02060000 and sys.version_info[0] == {{HOST_PY_MAJOR}}));" >/dev/null 2>&1 +do if "$py_cmd" -c "import sys; sys.exit(not (sys.version_info >= (2, 6) and sys.version_info[0] == {{HOST_PY_MAJOR}}));" then py_cmd_path=`"$py_cmd" -c 'from __future__ import print_function; import sys; print(sys.executable);'` exec $SUDO "$py_cmd_path" -c 'import base64; exec(base64.b64decode("""{{SSH_PY_CODE}}""").decode("utf-8"))' exit 0 @@ -245,7 +246,9 @@ class SSH(object): self.serial = salt.payload.Serial(opts) self.returners = salt.loader.returners(self.opts, {}) self.fsclient = salt.fileclient.FSClient(self.opts) - self.thin = salt.utils.thin.gen_thin(self.opts['cachedir']) + self.thin = salt.utils.thin.gen_thin(self.opts['cachedir'], + python2_bin=self.opts['python2_bin'], + python3_bin=self.opts['python3_bin']) self.mods = mod_data(self.fsclient) def get_pubkey(self): @@ -525,8 +528,11 @@ class SSH(object): # save load to the master job cache try: + if isinstance(jid, bytes): + jid = jid.decode('utf-8') self.returners['{0}.save_load'.format(self.opts['master_job_cache'])](jid, job_load) except Exception as exc: + log.exception(exc) log.error('Could not save load with returner {0}: {1}'.format(self.opts['master_job_cache'], exc)) if self.opts.get('verbose'): @@ -917,8 +923,10 @@ ARGS = {9}\n'''.format(self.minion_config, self.tty, self.argv) py_code = SSH_PY_SHIM.replace('#%%OPTS', arg_str) - py_code_enc = py_code.encode('base64') - + if six.PY2: + py_code_enc = py_code.encode('base64') + else: + py_code_enc = base64.encodebytes(py_code.encode('utf-8')).decode('utf-8') cmd = SSH_SH_SHIM.format( DEBUG=debug, SUDO=sudo, diff --git a/salt/runners/thin.py b/salt/runners/thin.py index 0a8b398a02..bec066d4ba 100644 --- a/salt/runners/thin.py +++ b/salt/runners/thin.py @@ -15,7 +15,8 @@ from __future__ import absolute_import import salt.utils.thin -def generate(extra_mods='', overwrite=False, so_mods=''): +def generate(extra_mods='', overwrite=False, so_mods='', + python2_bin='python2', python3_bin='python3'): ''' Generate the salt-thin tarball and print the location of the tarball Optional additional mods to include (e.g. mako) can be supplied as a comma @@ -30,4 +31,9 @@ def generate(extra_mods='', overwrite=False, so_mods=''): salt-run thin.generate mako,wempy 1 salt-run thin.generate overwrite=1 ''' - return salt.utils.thin.gen_thin(__opts__['cachedir'], extra_mods, overwrite, so_mods) + return salt.utils.thin.gen_thin(__opts__['cachedir'], + extra_mods, + overwrite, + so_mods, + python2_bin, + python3_bin) diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py index 7b71dff02d..eabe877c1e 100644 --- a/salt/utils/parsers.py +++ b/salt/utils/parsers.py @@ -2446,6 +2446,16 @@ class SaltSSHOptionParser(six.with_metaclass(OptionParserMeta, action='store_true', help=('Select a random temp dir to deploy on the remote system. ' 'The dir will be cleaned after the execution.')) + self.add_option( + '--python2-bin', + default='python2', + help='Path to a python2 binary which has salt installed' + ) + self.add_option( + '--python3-bin', + default='python3', + help='Path to a python3 binary which has salt installed' + ) auth_group = optparse.OptionGroup( self, 'Authentication Options', diff --git a/salt/utils/thin.py b/salt/utils/thin.py index 8d720e093f..af90f36800 100644 --- a/salt/utils/thin.py +++ b/salt/utils/thin.py @@ -7,16 +7,20 @@ Generate the salt thin tarball from the installed python files from __future__ import absolute_import import os +import sys +import json import shutil import tarfile import zipfile import tempfile +import subprocess # Import third party libs import jinja2 import yaml import salt.ext.six as six import tornado +import msgpack # pylint: disable=import-error,no-name-in-module try: @@ -56,6 +60,17 @@ import salt import salt.utils SALTCALL = ''' +import os +import sys + +sys.path.insert( + 0, + os.path.join( + os.path.dirname(__file__), + 'py{0[0]}'.format(sys.version_info) + ) +) + from salt.scripts import salt_call if __name__ == '__main__': salt_call() @@ -69,50 +84,13 @@ def thin_path(cachedir): return os.path.join(cachedir, 'thin', 'thin.tgz') -def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods=''): - ''' - Generate the salt-thin tarball and print the location of the tarball - Optional additional mods to include (e.g. mako) can be supplied as a comma - delimited string. Permits forcing an overwrite of the output file as well. - - CLI Example: - - .. code-block:: bash - - salt-run thin.generate - salt-run thin.generate mako - salt-run thin.generate mako,wempy 1 - salt-run thin.generate overwrite=1 - ''' - thindir = os.path.join(cachedir, 'thin') - if not os.path.isdir(thindir): - os.makedirs(thindir) - thintar = os.path.join(thindir, 'thin.tgz') - thinver = os.path.join(thindir, 'version') - salt_call = os.path.join(thindir, 'salt-call') - with salt.utils.fopen(salt_call, 'w+') as fp_: - fp_.write(SALTCALL) - if os.path.isfile(thintar): - if not overwrite: - if os.path.isfile(thinver): - with salt.utils.fopen(thinver) as fh_: - overwrite = fh_.read() != salt.version.__version__ - else: - overwrite = True - - if overwrite: - try: - os.remove(thintar) - except OSError: - pass - else: - return thintar - +def get_tops(extra_mods='', so_mods=''): tops = [ os.path.dirname(salt.__file__), os.path.dirname(jinja2.__file__), os.path.dirname(yaml.__file__), os.path.dirname(tornado.__file__), + os.path.dirname(msgpack.__file__) ] tops.append(six.__file__.replace('.pyc', '.py')) @@ -152,35 +130,120 @@ def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods=''): pass # As per comment above if HAS_MARKUPSAFE: tops.append(os.path.dirname(markupsafe.__file__)) + + return tops + + +def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='', + python2_bin='python2', python3_bin='python3'): + ''' + Generate the salt-thin tarball and print the location of the tarball + Optional additional mods to include (e.g. mako) can be supplied as a comma + delimited string. Permits forcing an overwrite of the output file as well. + + CLI Example: + + .. code-block:: bash + + salt-run thin.generate + salt-run thin.generate mako + salt-run thin.generate mako,wempy 1 + salt-run thin.generate overwrite=1 + ''' + thindir = os.path.join(cachedir, 'thin') + if not os.path.isdir(thindir): + os.makedirs(thindir) + thintar = os.path.join(thindir, 'thin.tgz') + thinver = os.path.join(thindir, 'version') + salt_call = os.path.join(thindir, 'salt-call') + with salt.utils.fopen(salt_call, 'w+') as fp_: + fp_.write(SALTCALL) + if os.path.isfile(thintar): + if not overwrite: + if os.path.isfile(thinver): + with salt.utils.fopen(thinver) as fh_: + overwrite = fh_.read() != salt.version.__version__ + else: + overwrite = True + + if overwrite: + try: + os.remove(thintar) + except OSError: + pass + else: + return thintar + + tops_py_version_mapping = {} + tops = get_tops(extra_mods=extra_mods, so_mods=so_mods) + if six.PY2: + tops_py_version_mapping['2'] = tops + else: + tops_py_version_mapping['3'] = tops + + if six.PY2 and sys.version_info[0] == 2: + # Get python 3 tops + py_shell_cmd = ( + python3_bin + ' -c \'import sys; import json; import salt.utils.thin; ' + 'print(json.dumps(salt.utils.thin.get_tops(**(json.loads(sys.argv[1]))))); exit(0);\' ' + '\'{0}\''.format(json.dumps({'extra_mods': extra_mods, 'so_mods': so_mods})) + ) + cmd = subprocess.Popen(py_shell_cmd, stdout=subprocess.PIPE, shell=True) + stdout, stderr = cmd.communicate() + if cmd.returncode == 0: + try: + tops = json.loads(stdout) + tops_py_version_mapping['3'] = tops + except ValueError: + pass + if six.PY3 and sys.version_info[0] == 3: + # Get python 2 tops + py_shell_cmd = ( + python2_bin + ' -c \'from __future__ import print_function; ' + 'import sys; import json; import salt.utils.thin; ' + 'print(json.dumps(salt.utils.thin.get_tops(**(json.loads(sys.argv[1]))))); exit(0);\' ' + '\'{0}\''.format(json.dumps({'extra_mods': extra_mods, 'so_mods': so_mods})) + ) + cmd = subprocess.Popen(py_shell_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + stdout, stderr = cmd.communicate() + if cmd.returncode == 0: + try: + tops = json.loads(stdout.decode('utf-8')) + tops_py_version_mapping['2'] = tops + except ValueError: + pass + tfp = tarfile.open(thintar, 'w:gz', dereference=True) try: # cwd may not exist if it was removed but salt was run from it start_dir = os.getcwd() except OSError: start_dir = None tempdir = None - for top in tops: - base = os.path.basename(top) - top_dirname = os.path.dirname(top) - if os.path.isdir(top_dirname): - os.chdir(top_dirname) - else: - # This is likely a compressed python .egg - tempdir = tempfile.mkdtemp() - egg = zipfile.ZipFile(top_dirname) - egg.extractall(tempdir) - top = os.path.join(tempdir, base) - os.chdir(tempdir) - if not os.path.isdir(top): - # top is a single file module - tfp.add(base) - continue - for root, dirs, files in os.walk(base, followlinks=True): - for name in files: - if not name.endswith(('.pyc', '.pyo')): - tfp.add(os.path.join(root, name)) - if tempdir is not None: - shutil.rmtree(tempdir) - tempdir = None + for py_ver, tops in six.iteritems(tops_py_version_mapping): + for top in tops: + base = os.path.basename(top) + top_dirname = os.path.dirname(top) + if os.path.isdir(top_dirname): + os.chdir(top_dirname) + else: + # This is likely a compressed python .egg + tempdir = tempfile.mkdtemp() + egg = zipfile.ZipFile(top_dirname) + egg.extractall(tempdir) + top = os.path.join(tempdir, base) + os.chdir(tempdir) + if not os.path.isdir(top): + # top is a single file module + tfp.add(base, arcname=os.path.join('py{0}'.format(py_ver), base)) + continue + for root, dirs, files in os.walk(base, followlinks=True): + for name in files: + if not name.endswith(('.pyc', '.pyo')): + tfp.add(os.path.join(root, name), + arcname=os.path.join('py{0}'.format(py_ver), root, name)) + if tempdir is not None: + shutil.rmtree(tempdir) + tempdir = None os.chdir(thindir) tfp.add('salt-call') with salt.utils.fopen(thinver, 'w+') as fp_: From 92e0e90e98b439175df41f862caf737baa05e9fc Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 13 Jul 2015 11:11:53 +0100 Subject: [PATCH 06/26] Remove duplicate entry --- doc/topics/ssh/roster.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/topics/ssh/roster.rst b/doc/topics/ssh/roster.rst index 762237653e..91aed8d802 100644 --- a/doc/topics/ssh/roster.rst +++ b/doc/topics/ssh/roster.rst @@ -45,7 +45,6 @@ The information which can be stored in a roster `target` is the following: priv: # File path to ssh private key, defaults to salt-ssh.rsa timeout: # Number of seconds to wait for response when establishing # an SSH connection - timeout: # Number of seconds to wait for response minion_opts: # Dictionary of minion opts thin_dir: # The target system's storage directory for Salt # components. Defaults to /tmp/salt-. From e508088a2ab01ffc97f1da359c93992553d6be4f Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 13 Jul 2015 18:51:52 +0100 Subject: [PATCH 07/26] Move `salt.config` to it's own package --- salt/{config.py => config/__init__.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename salt/{config.py => config/__init__.py} (100%) diff --git a/salt/config.py b/salt/config/__init__.py similarity index 100% rename from salt/config.py rename to salt/config/__init__.py From 8869c3ed1978f5cd166496823dbe2e2582b79c29 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 13 Jul 2015 18:54:26 +0100 Subject: [PATCH 08/26] Improvements to the config schema base classes * Don't include properties if empty * `DictConfig` now also accepts `Configuration` items. * Added `PortConfig` which sets some sane defaults for ports --- salt/utils/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/schema.py b/salt/utils/schema.py index fa11c18c7c..e346e3bf14 100644 --- a/salt/utils/schema.py +++ b/salt/utils/schema.py @@ -520,7 +520,7 @@ class Schema(six.with_metaclass(SchemaMeta, object)): # Define some class level attributes to make PyLint happier title = None description = None - _items = _sections = None + _items = _sections = _order = None __flatten__ = False __allow_additional_items__ = False From ddf814174e933277b899625fe49ba4bf227e2d8c Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 13 Jul 2015 18:56:38 +0100 Subject: [PATCH 09/26] Rename the config unit tests module --- tests/unit/{ => config}/config_test.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/unit/{ => config}/config_test.py (100%) diff --git a/tests/unit/config_test.py b/tests/unit/config/config_test.py similarity index 100% rename from tests/unit/config_test.py rename to tests/unit/config/config_test.py From eabc84d93c48dfa247c3581a2abbe81c437ab3fe Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 13 Jul 2015 19:34:21 +0100 Subject: [PATCH 10/26] Implemented `RosterEntryConfig` and added unit tests for it --- salt/config/schemas/__init__.py | 9 + salt/config/schemas/common.py | 63 +++++++ salt/config/schemas/minion.py | 35 ++++ salt/config/schemas/ssh.py | 71 ++++++++ tests/unit/config/__init__.py | 1 + tests/unit/config/schemas/__init__.py | 1 + tests/unit/config/schemas/test_ssh.py | 244 ++++++++++++++++++++++++++ 7 files changed, 424 insertions(+) create mode 100644 salt/config/schemas/__init__.py create mode 100644 salt/config/schemas/common.py create mode 100644 salt/config/schemas/minion.py create mode 100644 salt/config/schemas/ssh.py create mode 100644 tests/unit/config/__init__.py create mode 100644 tests/unit/config/schemas/__init__.py create mode 100644 tests/unit/config/schemas/test_ssh.py diff --git a/salt/config/schemas/__init__.py b/salt/config/schemas/__init__.py new file mode 100644 index 0000000000..5d301f75ba --- /dev/null +++ b/salt/config/schemas/__init__.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)` + + salt.config.schemas + ~~~~~~~~~~~~~~~~~~~ + + Salt configuration related schemas for future validation +''' diff --git a/salt/config/schemas/common.py b/salt/config/schemas/common.py new file mode 100644 index 0000000000..88573bfc4e --- /dev/null +++ b/salt/config/schemas/common.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)` + + + salt.config.schemas.common + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Common salt configuration schemas +''' + +# Import Pythosn libs +from __future__ import absolute_import + +# Import salt libs +from salt.utils.config import (Configuration, + StringConfig, + ArrayConfig, + OneOfConfig) + + +class DefaultIncludeConfig(StringConfig): + ''' + Per default, the {0}, will automatically include all config files + from '{1}/*.conf' ('{1}' is a sub-directory in the same directory + as the main {0} config file). + ''' + __target__ = None + __confd_directory__ = None + + title = 'Include Config' + description = __doc__ + + def __init__(self, default=None, pattern=None, **kwargs): + default = '{0}/*.conf'.format(self.__confd_directory__) + pattern = r'(?:.*)/\*\.conf' + super(DefaultIncludeConfig, self).__init__(default=default, pattern=pattern, **kwargs) + + def __validate_attributes__(self): + self.__doc__ = DefaultIncludeConfig.__doc__.format(self.__target__, + self.__confd_directory__) + super(DefaultIncludeConfig, self).__validate_attributes__() + + def __get_description__(self): + return self.__doc__.format(self.__target__, self.__confd_directory__) + + +class MinionDefaultInclude(DefaultIncludeConfig): + __target__ = 'minion' + __confd_directory__ = 'minion.d' + + +class MasterDefaultInclude(DefaultIncludeConfig): + __target__ = 'master' + __confd_directory = 'master.d' + + +class IncludeConfig(Configuration): + title = 'Include Configuration File(s)' + description = 'Include one or more specific configuration files' + + string_or_array = OneOfConfig(items=(StringConfig(), + ArrayConfig(items=StringConfig())))(flatten=True) diff --git a/salt/config/schemas/minion.py b/salt/config/schemas/minion.py new file mode 100644 index 0000000000..9eba2d324f --- /dev/null +++ b/salt/config/schemas/minion.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)` + + salt.config.schemas.minion + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Minion configuration schema +''' + +# Import python libs +from __future__ import absolute_import + +# Import salt libs +from salt.utils.config import (Configuration, + IPv4Config, + ) +from salt.config.schemas.common import (MinionDefaultInclude, + IncludeConfig + ) + +# XXX: THIS IS WAY TOO MINIMAL, BUT EXISTS TO IMPLEMENT salt-ssh + + +class MinionConfiguration(Configuration): + + # Because salt's configuration is very permissive with additioal + # configuration settings, let's allow them in the schema or validation + # would fail + __allow_additional_items__ = True + + interface = IPv4Config(title='Interface') + + default_include = MinionDefaultInclude() + include = IncludeConfig() diff --git a/salt/config/schemas/ssh.py b/salt/config/schemas/ssh.py new file mode 100644 index 0000000000..b3175631cf --- /dev/null +++ b/salt/config/schemas/ssh.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)` + + + salt.config.schemas.ssh + ~~~~~~~~~~~~~~~~~~~~~~~ + + Salt SSH related configuration schemas +''' + +# Import Python libs +from __future__ import absolute_import + +# Import Salt libs +from salt.utils.config import (Configuration, + StringConfig, + IntegerConfig, + SecretConfig, + OneOfConfig, + IPv4Config, + HostnameConfig, + PortConfig, + BooleanConfig, + RequirementsItem, + DictConfig, + AnyOfConfig + ) +from salt.config.schemas.minion import MinionConfiguration + + +class RosterEntryConfig(Configuration): + ''' + Schema definition of a Salt SSH Roster entry + ''' + + title = 'Roster Entry' + description = 'Salt SSH roster entry definition' + + host = StringConfig(title='Host', + description='The IP address or DNS name of the remote host', + # Pretty naive pattern matching + pattern=r'^((\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|([A-Za-z0-9][A-Za-z0-9\.\-]{1,255}))$', + min_length=1, + required=True) + port = PortConfig(title='Port', + description='The target system\'s ssh port number', + default=22) + user = StringConfig(title='User', + description='The user to log in as. Defaults to root', + default='root', + min_length=1, + required=True) + passwd = SecretConfig(title='Password', + description='The password to log in with') + priv = StringConfig(title='Private Key', + description='File path to ssh private key, defaults to salt-ssh.rsa') + passwd_or_priv_requirement = AnyOfConfig(items=(RequirementsItem(requirements=['passwd']), + RequirementsItem(requirements=['priv'])))(flatten=True) + sudo = BooleanConfig(title='Sudo', + description='run command via sudo. Defaults to False', + default=False) + timeout = IntegerConfig(title='Timeout', + description=('Number of seconds to wait for response ' + 'when establishing an SSH connection')) + thin_dir = StringConfig(title='Thin Directory', + description=('The target system\'s storage directory for Salt ' + 'components. Defaults to /tmp/salt-.')) + minion_opts = DictConfig(title='Minion Options', + description='Dictionary of minion options', + properties=MinionConfiguration()) diff --git a/tests/unit/config/__init__.py b/tests/unit/config/__init__.py new file mode 100644 index 0000000000..40a96afc6f --- /dev/null +++ b/tests/unit/config/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/tests/unit/config/schemas/__init__.py b/tests/unit/config/schemas/__init__.py new file mode 100644 index 0000000000..40a96afc6f --- /dev/null +++ b/tests/unit/config/schemas/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/tests/unit/config/schemas/test_ssh.py b/tests/unit/config/schemas/test_ssh.py new file mode 100644 index 0000000000..bc5ec9e45e --- /dev/null +++ b/tests/unit/config/schemas/test_ssh.py @@ -0,0 +1,244 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)` + + tests.unit.config.schemas.test_ssh + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +''' +# Import python libs +from __future__ import absolute_import + +# Import Salt Testing Libs +from salttesting import TestCase, skipIf +from salttesting.helpers import ensure_in_syspath + +ensure_in_syspath('../../') + +# Import Salt Libs +from salt.config.schemas import ssh as ssh_schemas +from salt.config.schemas.minion import MinionConfiguration +from salt.utils.config import DictConfig + +# Import 3rd-party libs +try: + import jsonschema + import jsonschema.exceptions + HAS_JSONSCHEMA = True +except ImportError: + HAS_JSONSCHEMA = False + + +class RoosterEntryConfigTest(TestCase): + def test_config(self): + config = ssh_schemas.RosterEntryConfig() + + expected = { + '$schema': 'http://json-schema.org/draft-04/schema#', + 'title': 'Roster Entry', + 'description': 'Salt SSH roster entry definition', + 'type': 'object', + 'properties': { + 'host': { + 'title': 'Host', + 'description': 'The IP address or DNS name of the remote host', + 'type': 'string', + 'pattern': r'^((\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|([A-Za-z0-9][A-Za-z0-9\.\-]{1,255}))$', + 'minLength': 1 + }, + 'port': { + 'description': 'The target system\'s ssh port number', + 'title': 'Port', + 'default': 22, + 'maximum': 65535, + 'minimum': 0, + 'type': 'integer' + }, + 'user': { + 'default': 'root', + 'type': 'string', + 'description': 'The user to log in as. Defaults to root', + 'title': 'User', + 'minLength': 1 + }, + 'passwd': { + 'title': 'Password', + 'type': 'string', + 'description': 'The password to log in with', + 'format': 'secret' + }, + 'priv': { + 'type': 'string', + 'description': 'File path to ssh private key, defaults to salt-ssh.rsa', + 'title': 'Private Key' + }, + 'sudo': { + 'default': False, + 'type': 'boolean', + 'description': 'run command via sudo. Defaults to False', + 'title': 'Sudo' + }, + 'timeout': { + 'type': 'integer', + 'description': 'Number of seconds to wait for response when establishing an SSH connection', + 'title': 'Timeout' + }, + 'thin_dir': { + 'type': 'string', + 'description': 'The target system\'s storage directory for Salt components. Defaults to /tmp/salt-.', + 'title': 'Thin Directory' + }, + # The actuall representation of the minion options would make this HUGE! + 'minion_opts': DictConfig(title='Minion Options', + description='Dictionary of minion options', + properties=MinionConfiguration()).serialize(), + }, + 'anyOf': [ + { + 'required': [ + 'passwd' + ] + }, + { + 'required': [ + 'priv' + ] + } + ], + 'required': [ + 'host', + 'user', + ], + 'x-ordering': [ + 'host', + 'port', + 'user', + 'passwd', + 'priv', + 'sudo', + 'timeout', + 'thin_dir', + 'minion_opts' + ], + 'additionalProperties': False + } + try: + self.assertDictContainsSubset(expected['properties'], config.serialize()['properties']) + self.assertDictContainsSubset(expected, config.serialize()) + except AssertionError: + import json + print(json.dumps(config.serialize(), indent=4)) + raise + + @skipIf(HAS_JSONSCHEMA is False, 'The \'jsonschema\' library is missing') + def test_config_validate(self): + try: + jsonschema.validate( + { + 'host': 'localhost', + 'user': 'root', + 'passwd': 'foo' + }, + ssh_schemas.RosterEntryConfig.serialize(), + format_checker=jsonschema.FormatChecker() + ) + except jsonschema.exceptions.ValidationError as exc: + self.fail('ValidationError raised: {0}'.format(exc)) + + try: + jsonschema.validate( + { + 'host': '127.0.0.1', + 'user': 'root', + 'passwd': 'foo' + }, + ssh_schemas.RosterEntryConfig.serialize(), + format_checker=jsonschema.FormatChecker() + ) + except jsonschema.exceptions.ValidationError as exc: + self.fail('ValidationError raised: {0}'.format(exc)) + + try: + jsonschema.validate( + { + 'host': '127.1.0.1', + 'user': 'root', + 'priv': 'foo', + 'passwd': 'foo' + }, + ssh_schemas.RosterEntryConfig.serialize(), + format_checker=jsonschema.FormatChecker() + ) + except jsonschema.exceptions.ValidationError as exc: + self.fail('ValidationError raised: {0}'.format(exc)) + + try: + jsonschema.validate( + { + 'host': '127.1.0.1', + 'user': 'root', + 'passwd': 'foo', + 'sudo': False + }, + ssh_schemas.RosterEntryConfig.serialize(), + format_checker=jsonschema.FormatChecker() + ) + except jsonschema.exceptions.ValidationError as exc: + self.fail('ValidationError raised: {0}'.format(exc)) + + try: + jsonschema.validate( + { + 'host': '127.1.0.1', + 'user': 'root', + 'priv': 'foo', + 'passwd': 'foo', + 'thin_dir': '/foo/bar' + }, + ssh_schemas.RosterEntryConfig.serialize(), + format_checker=jsonschema.FormatChecker() + ) + except jsonschema.exceptions.ValidationError as exc: + self.fail('ValidationError raised: {0}'.format(exc)) + + try: + jsonschema.validate( + { + 'host': '127.1.0.1', + 'user': 'root', + 'passwd': 'foo', + 'minion_opts': { + 'interface': '0.0.0.0' + } + }, + ssh_schemas.RosterEntryConfig.serialize(), + format_checker=jsonschema.FormatChecker() + ) + except jsonschema.exceptions.ValidationError as exc: + self.fail('ValidationError raised: {0}'.format(exc)) + + with self.assertRaises(jsonschema.exceptions.ValidationError) as excinfo: + jsonschema.validate( + { + 'host': '127.1.0.1', + 'user': '', + 'passwd': 'foo', + }, + ssh_schemas.RosterEntryConfig.serialize(), + format_checker=jsonschema.FormatChecker() + ) + self.assertIn('is too short', excinfo.exception.message) + + with self.assertRaises(jsonschema.exceptions.ValidationError) as excinfo: + jsonschema.validate( + { + 'host': '127.1.0.1', + 'user': 'root', + 'passwd': 'foo', + 'minion_opts': { + 'interface': 0 + } + }, + ssh_schemas.RosterEntryConfig.serialize(), + format_checker=jsonschema.FormatChecker() + ) + self.assertIn('is not of type', excinfo.exception.message) From 63cecaddf4c5a664a443e1f35210157b220bf8bc Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 14 Jul 2015 00:24:44 +0100 Subject: [PATCH 11/26] Implement the `Roster` config schema --- salt/config/schemas/ssh.py | 9 +++++ tests/unit/config/schemas/test_ssh.py | 57 +++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/salt/config/schemas/ssh.py b/salt/config/schemas/ssh.py index b3175631cf..485daccbca 100644 --- a/salt/config/schemas/ssh.py +++ b/salt/config/schemas/ssh.py @@ -69,3 +69,12 @@ class RosterEntryConfig(Configuration): minion_opts = DictConfig(title='Minion Options', description='Dictionary of minion options', properties=MinionConfiguration()) + + +class RosterConfig(Configuration): + title = 'Roster Configuration' + description = 'Roster entries definition' + + roster_entries = DictConfig( + pattern_properties={ + r'^([^:]+)$': RosterEntryConfig()})(flatten=True) diff --git a/tests/unit/config/schemas/test_ssh.py b/tests/unit/config/schemas/test_ssh.py index bc5ec9e45e..68eb376067 100644 --- a/tests/unit/config/schemas/test_ssh.py +++ b/tests/unit/config/schemas/test_ssh.py @@ -242,3 +242,60 @@ class RoosterEntryConfigTest(TestCase): format_checker=jsonschema.FormatChecker() ) self.assertIn('is not of type', excinfo.exception.message) + + +class RosterConfigTest(TestCase): + + def test_roster_config(self): + try: + self.assertDictEqual( + { + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "roster_entries", + "description": "Roster entries definition", + "type": "object", + "patternProperties": { + r"^([^:]+)$": ssh_schemas.RosterEntryConfig.serialize() + }, + "additionalProperties": False + }, + ssh_schemas.RosterConfig.serialize() + ) + except AssertionError: + import json + print(json.dumps(ssh_schemas.RosterConfig.serialize(), indent=4)) + raise + + @skipIf(HAS_JSONSCHEMA is False, 'The \'jsonschema\' library is missing') + def test_roster_config_validate(self): + try: + jsonschema.validate( + {'target-1': + { + 'host': 'localhost', + 'user': 'root', + 'passwd': 'foo' + } + }, + ssh_schemas.RosterConfig.serialize(), + format_checker=jsonschema.FormatChecker() + ) + except jsonschema.exceptions.ValidationError as exc: + self.fail('ValidationError raised: {0}'.format(exc)) + + with self.assertRaises(jsonschema.exceptions.ValidationError) as excinfo: + jsonschema.validate( + {'target-1:1': + { + 'host': 'localhost', + 'user': 'root', + 'passwd': 'foo' + } + }, + ssh_schemas.RosterConfig.serialize(), + format_checker=jsonschema.FormatChecker() + ) + self.assertIn( + 'Additional properties are not allowed (\'target-1:1\' was unexpected)', + excinfo.exception.message + ) From 41f60e4eaaa2a429095239807528ced233449d00 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 15 Jul 2015 14:51:54 +0100 Subject: [PATCH 12/26] Store the python version used to create the thin. Force re-generation of the thin if using a different python version. Python version comparison is only done between python major versions 2 and 3. --- salt/utils/thin.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/utils/thin.py b/salt/utils/thin.py index af90f36800..c0687cf534 100644 --- a/salt/utils/thin.py +++ b/salt/utils/thin.py @@ -155,6 +155,7 @@ def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='', os.makedirs(thindir) thintar = os.path.join(thindir, 'thin.tgz') thinver = os.path.join(thindir, 'version') + pythinver = os.path.join(thindir, '.thin-gen-py-version') salt_call = os.path.join(thindir, 'salt-call') with salt.utils.fopen(salt_call, 'w+') as fp_: fp_.write(SALTCALL) @@ -163,6 +164,9 @@ def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='', if os.path.isfile(thinver): with salt.utils.fopen(thinver) as fh_: overwrite = fh_.read() != salt.version.__version__ + if overwrite is False and os.path.isfile(pythinver): + with salt.utils.fopen(pythinver) as fh_: + overwrite = fh_.read() != str(sys.version_info[0]) else: overwrite = True @@ -248,8 +252,11 @@ def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='', tfp.add('salt-call') with salt.utils.fopen(thinver, 'w+') as fp_: fp_.write(salt.version.__version__) + with salt.utils.fopen(pythinver, 'w+') as fp_: + fp_.write(str(sys.version_info[0])) os.chdir(os.path.dirname(thinver)) tfp.add('version') + tfp.add('.thin-gen-py-version') if start_dir: os.chdir(start_dir) tfp.close() From 8fa8d5c5b584525fc7ff0725e0d0adaf6e8f4ca6 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 15 Jul 2015 14:56:56 +0100 Subject: [PATCH 13/26] Add TODO comment --- salt/utils/thin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/utils/thin.py b/salt/utils/thin.py index c0687cf534..b9dff07673 100644 --- a/salt/utils/thin.py +++ b/salt/utils/thin.py @@ -185,6 +185,8 @@ def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='', else: tops_py_version_mapping['3'] = tops + # TODO: Consider putting know py2 and py3 compatible libs in it's own sharable directory. + # This would reduce the thin size. if six.PY2 and sys.version_info[0] == 2: # Get python 3 tops py_shell_cmd = ( From ad95c0e8cffc456a9f9a7017b8ab42c77601a7e5 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 15 Jul 2015 20:30:59 +0100 Subject: [PATCH 14/26] msgpack data is binary, open in binary mode --- salt/utils/http.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/utils/http.py b/salt/utils/http.py index 21f11f0660..73c78d0ed2 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -218,12 +218,12 @@ def query(url, # proper cookie jar. Unfortunately, since session cookies do not # contain expirations, they can't be stored in a proper cookie jar. if os.path.isfile(session_cookie_jar): - with salt.utils.fopen(session_cookie_jar, 'r') as fh_: + with salt.utils.fopen(session_cookie_jar, 'rb') as fh_: session_cookies = msgpack.load(fh_) if isinstance(session_cookies, dict): header_dict.update(session_cookies) else: - with salt.utils.fopen(session_cookie_jar, 'w') as fh_: + with salt.utils.fopen(session_cookie_jar, 'wb') as fh_: msgpack.dump('', fh_) for header in header_list: @@ -462,7 +462,7 @@ def query(url, if persist_session is True and HAS_MSGPACK: # TODO: See persist_session above if 'set-cookie' in result_headers: - with salt.utils.fopen(session_cookie_jar, 'w') as fh_: + with salt.utils.fopen(session_cookie_jar, 'wb') as fh_: session_cookies = result_headers.get('set-cookie', None) if session_cookies is not None: msgpack.dump({'Cookie': session_cookies}, fh_) From 143e8fa20029bdbf1a37aa8ff63bdb8fc4151bcf Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 16 Jul 2015 17:01:42 +0100 Subject: [PATCH 15/26] Let's call it for what it is! --- tests/unit/utils/config_test.py | 249 ++++++++++++++++---------------- 1 file changed, 123 insertions(+), 126 deletions(-) diff --git a/tests/unit/utils/config_test.py b/tests/unit/utils/config_test.py index b35bc376af..78ef0941d4 100644 --- a/tests/unit/utils/config_test.py +++ b/tests/unit/utils/config_test.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# pylint: disable=function-redefined +# pylint: disable=function-redefined,missing-docstring # TODO: Remove the following PyLint disable as soon as we support YAML and RST rendering # pylint: disable=abstract-method @@ -67,8 +67,8 @@ class ConfigTestCase(TestCase): 'default': True, 'type': 'boolean', 'title': 'base' - }, - 'hungry': { + }, + 'hungry': { 'type': 'boolean', 'description': 'Are you hungry?', 'title': 'Hungry' @@ -519,8 +519,8 @@ class ConfigTestCase(TestCase): ) item = schema.BooleanItem(title='Hungry', - description='Are you hungry?', - default=False) + description='Are you hungry?', + default=False) self.assertDictEqual( item.serialize(), { 'type': 'boolean', @@ -531,8 +531,8 @@ class ConfigTestCase(TestCase): ) item = schema.BooleanItem(title='Hungry', - description='Are you hungry?', - default=schema.Null) + description='Are you hungry?', + default=schema.Null) self.assertDictEqual( item.serialize(), { 'type': 'boolean', @@ -578,10 +578,10 @@ class ConfigTestCase(TestCase): ) item = schema.StringItem(title='Foo', - description='Foo Item', - min_length=1, - max_length=3, - default='foo') + description='Foo Item', + min_length=1, + max_length=3, + default='foo') self.assertDictEqual( item.serialize(), { 'type': 'string', @@ -594,10 +594,10 @@ class ConfigTestCase(TestCase): ) item = schema.StringItem(title='Foo', - description='Foo Item', - min_length=1, - max_length=3, - enum=('foo', 'bar')) + description='Foo Item', + min_length=1, + max_length=3, + enum=('foo', 'bar')) self.assertDictEqual( item.serialize(), { 'type': 'string', @@ -610,11 +610,11 @@ class ConfigTestCase(TestCase): ) item = schema.StringItem(title='Foo', - description='Foo Item', - min_length=1, - max_length=3, - enum=('foo', 'bar'), - enumNames=('Foo', 'Bar')) + description='Foo Item', + min_length=1, + max_length=3, + enum=('foo', 'bar'), + enumNames=('Foo', 'Bar')) self.assertDictEqual( item.serialize(), { 'type': 'string', @@ -628,8 +628,8 @@ class ConfigTestCase(TestCase): ) item = schema.StringItem(title='Foo', - description='Foo Item', - pattern=r'^([\w_-]+)$') + description='Foo Item', + pattern=r'^([\w_-]+)$') self.assertDictEqual( item.serialize(), { 'type': 'string', @@ -651,7 +651,7 @@ class ConfigTestCase(TestCase): class TestConf(schema.Schema): item = schema.StringItem(title='Foo', description='Foo Item', - min_length=1, max_length=10) + min_length=1, max_length=10) try: jsonschema.validate({'item': 'the item'}, TestConf.serialize()) @@ -668,7 +668,7 @@ class ConfigTestCase(TestCase): class TestConf(schema.Schema): item = schema.StringItem(title='Foo', description='Foo Item', - min_length=10, max_length=100) + min_length=10, max_length=100) with self.assertRaises(jsonschema.exceptions.ValidationError) as excinfo: jsonschema.validate({'item': 'the item'}, TestConf.serialize()) @@ -676,8 +676,8 @@ class ConfigTestCase(TestCase): class TestConf(schema.Schema): item = schema.StringItem(title='Foo', - description='Foo Item', - enum=('foo', 'bar')) + description='Foo Item', + enum=('foo', 'bar')) try: jsonschema.validate({'item': 'foo'}, TestConf.serialize()) @@ -686,15 +686,15 @@ class ConfigTestCase(TestCase): class TestConf(schema.Schema): item = schema.StringItem(title='Foo', - description='Foo Item', - enum=('foo', 'bar')) + description='Foo Item', + enum=('foo', 'bar')) with self.assertRaises(jsonschema.exceptions.ValidationError) as excinfo: jsonschema.validate({'item': 'bin'}, TestConf.serialize()) self.assertIn('is not one of', excinfo.exception.message) class TestConf(schema.Schema): item = schema.StringItem(title='Foo', description='Foo Item', - pattern=r'^([\w_-]+)$') + pattern=r'^([\w_-]+)$') try: jsonschema.validate({'item': 'the-item'}, TestConf.serialize(), @@ -893,9 +893,9 @@ class ConfigTestCase(TestCase): ) item = schema.NumberItem(title='How many dogs', - description='Question', - minimum=0, - maximum=10) + description='Question', + minimum=0, + maximum=10) self.assertDictEqual( item.serialize(), { 'type': 'number', @@ -907,8 +907,8 @@ class ConfigTestCase(TestCase): ) item = schema.NumberItem(title='How many dogs', - description='Question', - multiple_of=2) + description='Question', + multiple_of=2) self.assertDictEqual( item.serialize(), { 'type': 'number', @@ -919,11 +919,11 @@ class ConfigTestCase(TestCase): ) item = schema.NumberItem(title='How many dogs', - description='Question', - minimum=0, - exclusive_minimum=True, - maximum=10, - exclusive_maximum=True) + description='Question', + minimum=0, + exclusive_minimum=True, + maximum=10, + exclusive_maximum=True) self.assertDictEqual( item.serialize(), { 'type': 'number', @@ -937,10 +937,10 @@ class ConfigTestCase(TestCase): ) item = schema.NumberItem(title='How many dogs', - description='Question', - minimum=0, - maximum=10, - default=0) + description='Question', + minimum=0, + maximum=10, + default=0) self.assertDictEqual( item.serialize(), { 'type': 'number', @@ -953,11 +953,11 @@ class ConfigTestCase(TestCase): ) item = schema.NumberItem(title='How many dogs', - description='Question', - minimum=0, - maximum=10, - default=0, - enum=(0, 2, 4, 6)) + description='Question', + minimum=0, + maximum=10, + default=0, + enum=(0, 2, 4, 6)) self.assertDictEqual( item.serialize(), { 'type': 'number', @@ -986,8 +986,8 @@ class ConfigTestCase(TestCase): class TestConf(schema.Schema): item = schema.NumberItem(title='How many dogs', - description='Question', - multiple_of=2.2) + description='Question', + multiple_of=2.2) try: jsonschema.validate({'item': 4.4}, TestConf.serialize()) @@ -1000,7 +1000,7 @@ class ConfigTestCase(TestCase): class TestConf(schema.Schema): item = schema.NumberItem(title='Foo', description='Foo Item', - minimum=1, maximum=10) + minimum=1, maximum=10) try: jsonschema.validate({'item': 3}, TestConf.serialize()) @@ -1013,7 +1013,7 @@ class ConfigTestCase(TestCase): class TestConf(schema.Schema): item = schema.NumberItem(title='Foo', description='Foo Item', - minimum=10, maximum=100) + minimum=10, maximum=100) with self.assertRaises(jsonschema.exceptions.ValidationError) as excinfo: jsonschema.validate({'item': 3}, TestConf.serialize()) @@ -1021,11 +1021,11 @@ class ConfigTestCase(TestCase): class TestConf(schema.Schema): item = schema.NumberItem(title='How many dogs', - description='Question', - minimum=0, - exclusive_minimum=True, - maximum=10, - exclusive_maximum=True) + description='Question', + minimum=0, + exclusive_minimum=True, + maximum=10, + exclusive_maximum=True) with self.assertRaises(jsonschema.exceptions.ValidationError) as excinfo: jsonschema.validate({'item': 0}, TestConf.serialize()) @@ -1037,8 +1037,8 @@ class ConfigTestCase(TestCase): class TestConf(schema.Schema): item = schema.NumberItem(title='Foo', - description='Foo Item', - enum=(0, 2, 4, 6)) + description='Foo Item', + enum=(0, 2, 4, 6)) try: jsonschema.validate({'item': 4}, TestConf.serialize()) @@ -1047,8 +1047,8 @@ class ConfigTestCase(TestCase): class TestConf(schema.Schema): item = schema.NumberItem(title='Foo', - description='Foo Item', - enum=(0, 2, 4, 6)) + description='Foo Item', + enum=(0, 2, 4, 6)) with self.assertRaises(jsonschema.exceptions.ValidationError) as excinfo: jsonschema.validate({'item': 3}, TestConf.serialize()) self.assertIn('is not one of', excinfo.exception.message) @@ -1064,9 +1064,9 @@ class ConfigTestCase(TestCase): ) item = schema.IntegerItem(title='How many dogs', - description='Question', - minimum=0, - maximum=10) + description='Question', + minimum=0, + maximum=10) self.assertDictEqual( item.serialize(), { 'type': 'integer', @@ -1078,8 +1078,8 @@ class ConfigTestCase(TestCase): ) item = schema.IntegerItem(title='How many dogs', - description='Question', - multiple_of=2) + description='Question', + multiple_of=2) self.assertDictEqual( item.serialize(), { 'type': 'integer', @@ -1090,11 +1090,11 @@ class ConfigTestCase(TestCase): ) item = schema.IntegerItem(title='How many dogs', - description='Question', - minimum=0, - exclusive_minimum=True, - maximum=10, - exclusive_maximum=True) + description='Question', + minimum=0, + exclusive_minimum=True, + maximum=10, + exclusive_maximum=True) self.assertDictEqual( item.serialize(), { 'type': 'integer', @@ -1108,10 +1108,10 @@ class ConfigTestCase(TestCase): ) item = schema.IntegerItem(title='How many dogs', - description='Question', - minimum=0, - maximum=10, - default=0) + description='Question', + minimum=0, + maximum=10, + default=0) self.assertDictEqual( item.serialize(), { 'type': 'integer', @@ -1124,11 +1124,11 @@ class ConfigTestCase(TestCase): ) item = schema.IntegerItem(title='How many dogs', - description='Question', - minimum=0, - maximum=10, - default=0, - enum=(0, 2, 4, 6)) + description='Question', + minimum=0, + maximum=10, + default=0, + enum=(0, 2, 4, 6)) self.assertDictEqual( item.serialize(), { 'type': 'integer', @@ -1157,8 +1157,8 @@ class ConfigTestCase(TestCase): class TestConf(schema.Schema): item = schema.IntegerItem(title='How many dogs', - description='Question', - multiple_of=2) + description='Question', + multiple_of=2) try: jsonschema.validate({'item': 4}, TestConf.serialize()) @@ -1171,7 +1171,7 @@ class ConfigTestCase(TestCase): class TestConf(schema.Schema): item = schema.IntegerItem(title='Foo', description='Foo Item', - minimum=1, maximum=10) + minimum=1, maximum=10) try: jsonschema.validate({'item': 3}, TestConf.serialize()) @@ -1184,7 +1184,7 @@ class ConfigTestCase(TestCase): class TestConf(schema.Schema): item = schema.IntegerItem(title='Foo', description='Foo Item', - minimum=10, maximum=100) + minimum=10, maximum=100) with self.assertRaises(jsonschema.exceptions.ValidationError) as excinfo: jsonschema.validate({'item': 3}, TestConf.serialize()) @@ -1192,11 +1192,11 @@ class ConfigTestCase(TestCase): class TestConf(schema.Schema): item = schema.IntegerItem(title='How many dogs', - description='Question', - minimum=0, - exclusive_minimum=True, - maximum=10, - exclusive_maximum=True) + description='Question', + minimum=0, + exclusive_minimum=True, + maximum=10, + exclusive_maximum=True) with self.assertRaises(jsonschema.exceptions.ValidationError) as excinfo: jsonschema.validate({'item': 0}, TestConf.serialize()) @@ -1208,8 +1208,8 @@ class ConfigTestCase(TestCase): class TestConf(schema.Schema): item = schema.IntegerItem(title='Foo', - description='Foo Item', - enum=(0, 2, 4, 6)) + description='Foo Item', + enum=(0, 2, 4, 6)) try: jsonschema.validate({'item': 4}, TestConf.serialize()) @@ -1218,18 +1218,18 @@ class ConfigTestCase(TestCase): class TestConf(schema.Schema): item = schema.IntegerItem(title='Foo', - description='Foo Item', - enum=(0, 2, 4, 6)) + description='Foo Item', + enum=(0, 2, 4, 6)) with self.assertRaises(jsonschema.exceptions.ValidationError) as excinfo: jsonschema.validate({'item': 3}, TestConf.serialize()) self.assertIn('is not one of', excinfo.exception.message) def test_array_config(self): string_item = schema.StringItem(title='Dog Name', - description='The dog name') + description='The dog name') item = schema.ArrayItem(title='Dog Names', - description='Name your dogs', - items=string_item) + description='Name your dogs', + items=string_item) self.assertDictEqual( item.serialize(), { @@ -1245,10 +1245,10 @@ class ConfigTestCase(TestCase): ) integer_item = schema.IntegerItem(title='Dog Age', - description='The dog age') + description='The dog age') item = schema.ArrayItem(title='Dog Names', - description='Name your dogs', - items=(string_item, integer_item)) + description='Name your dogs', + items=(string_item, integer_item)) self.assertDictEqual( item.serialize(), { @@ -1271,13 +1271,13 @@ class ConfigTestCase(TestCase): ) item = schema.ArrayItem(title='Dog Names', - description='Name your dogs', - items=(schema.StringItem(), - schema.IntegerItem()), - min_items=1, - max_items=3, - additional_items=False, - unique_items=True) + description='Name your dogs', + items=(schema.StringItem(), + schema.IntegerItem()), + min_items=1, + max_items=3, + additional_items=False, + unique_items=True) self.assertDictEqual( item.serialize(), { @@ -1303,8 +1303,8 @@ class ConfigTestCase(TestCase): item = schema.IntegerItem(title='How many dogs', description='Question') item = schema.ArrayItem(title='Dog Names', - description='Name your dogs', - items=HowManyConfig()) + description='Name your dogs', + items=HowManyConfig()) self.assertDictEqual( item.serialize(), { 'type': 'array', @@ -1318,8 +1318,8 @@ class ConfigTestCase(TestCase): item = schema.IntegerItem() item = schema.ArrayItem(title='Dog Names', - description='Name your dogs', - items=(HowManyConfig(), AgesConfig())) + description='Name your dogs', + items=(HowManyConfig(), AgesConfig())) self.assertDictEqual( item.serialize(), { 'type': 'array', @@ -1336,8 +1336,8 @@ class ConfigTestCase(TestCase): def test_array_config_validation(self): class TestConf(schema.Schema): item = schema.ArrayItem(title='Dog Names', - description='Name your dogs', - items=schema.StringItem()) + description='Name your dogs', + items=schema.StringItem()) try: jsonschema.validate({'item': ['Tobias', 'Óscar']}, TestConf.serialize(), @@ -1352,10 +1352,10 @@ class ConfigTestCase(TestCase): class TestConf(schema.Schema): item = schema.ArrayItem(title='Dog Names', - description='Name your dogs', - items=schema.StringItem(), - min_items=1, - max_items=2) + description='Name your dogs', + items=schema.StringItem(), + min_items=1, + max_items=2) try: jsonschema.validate({'item': ['Tobias', 'Óscar']}, TestConf.serialize(), @@ -1375,9 +1375,9 @@ class ConfigTestCase(TestCase): class TestConf(schema.Schema): item = schema.ArrayItem(title='Dog Names', - description='Name your dogs', - items=schema.StringItem(), - uniqueItems=True) + description='Name your dogs', + items=schema.StringItem(), + uniqueItems=True) with self.assertRaises(jsonschema.exceptions.ValidationError) as excinfo: jsonschema.validate({'item': ['Tobias', 'Tobias']}, TestConf.serialize(), @@ -1386,7 +1386,7 @@ class ConfigTestCase(TestCase): class TestConf(schema.Schema): item = schema.ArrayItem(items=(schema.StringItem(), - schema.IntegerItem())) + schema.IntegerItem())) try: jsonschema.validate({'item': ['Óscar', 4]}, TestConf.serialize(), format_checker=jsonschema.FormatChecker()) @@ -1480,7 +1480,7 @@ class ConfigTestCase(TestCase): title='Poligon', description='Describe the Poligon', pattern_properties={ - 's*': schema.IntegerItem() + 's.*': schema.IntegerItem() }, min_properties=1, max_properties=2 @@ -1491,7 +1491,7 @@ class ConfigTestCase(TestCase): 'title': item.title, 'description': item.description, 'patternProperties': { - 's*': {'type': 'integer'} + 's.*': {'type': 'integer'} }, 'minProperties': 1, 'maxProperties': 2 @@ -1568,7 +1568,7 @@ class ConfigTestCase(TestCase): 'sides': schema.IntegerItem() }, additional_properties=schema.OneOfItem(items=[schema.BooleanItem(), - schema.StringItem()]) + schema.StringItem()]) ) self.assertDictEqual( item.serialize(), { @@ -1597,7 +1597,6 @@ class ConfigTestCase(TestCase): 'sides': schema.IntegerItem() } ) - try: jsonschema.validate({'item': {'sides': 1}}, TestConf.serialize()) except jsonschema.exceptions.ValidationError as exc: @@ -1663,7 +1662,6 @@ class ConfigTestCase(TestCase): additional_properties=schema.OneOfItem(items=[ schema.BooleanItem(), schema.IntegerItem() - ]) ) @@ -1688,7 +1686,6 @@ class ConfigTestCase(TestCase): additional_properties=schema.OneOfItem(items=[ schema.BooleanItem(), schema.IntegerItem() - ]), min_properties=2, max_properties=3 From 7a4d96d83af7c68adc61b946d434c66f875bd345 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 17 Jul 2015 16:03:20 +0100 Subject: [PATCH 16/26] Adapt to the module rename --- salt/config/schemas/common.py | 16 ++++++------ salt/config/schemas/minion.py | 8 +++--- salt/config/schemas/ssh.py | 48 +++++++++++++++++------------------ 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/salt/config/schemas/common.py b/salt/config/schemas/common.py index 88573bfc4e..ff74bc1589 100644 --- a/salt/config/schemas/common.py +++ b/salt/config/schemas/common.py @@ -13,13 +13,13 @@ from __future__ import absolute_import # Import salt libs -from salt.utils.config import (Configuration, - StringConfig, - ArrayConfig, - OneOfConfig) +from salt.utils.schema import (Schema, + StringItem, + ArrayItem, + OneOfItem) -class DefaultIncludeConfig(StringConfig): +class DefaultIncludeConfig(StringItem): ''' Per default, the {0}, will automatically include all config files from '{1}/*.conf' ('{1}' is a sub-directory in the same directory @@ -55,9 +55,9 @@ class MasterDefaultInclude(DefaultIncludeConfig): __confd_directory = 'master.d' -class IncludeConfig(Configuration): +class IncludeConfig(Schema): title = 'Include Configuration File(s)' description = 'Include one or more specific configuration files' - string_or_array = OneOfConfig(items=(StringConfig(), - ArrayConfig(items=StringConfig())))(flatten=True) + string_or_array = OneOfItem(items=(StringItem(), + ArrayItem(items=StringItem())))(flatten=True) diff --git a/salt/config/schemas/minion.py b/salt/config/schemas/minion.py index 9eba2d324f..74e626ba42 100644 --- a/salt/config/schemas/minion.py +++ b/salt/config/schemas/minion.py @@ -12,8 +12,8 @@ from __future__ import absolute_import # Import salt libs -from salt.utils.config import (Configuration, - IPv4Config, +from salt.utils.schema import (Schema, + IPv4Item, ) from salt.config.schemas.common import (MinionDefaultInclude, IncludeConfig @@ -22,14 +22,14 @@ from salt.config.schemas.common import (MinionDefaultInclude, # XXX: THIS IS WAY TOO MINIMAL, BUT EXISTS TO IMPLEMENT salt-ssh -class MinionConfiguration(Configuration): +class MinionConfiguration(Schema): # Because salt's configuration is very permissive with additioal # configuration settings, let's allow them in the schema or validation # would fail __allow_additional_items__ = True - interface = IPv4Config(title='Interface') + interface = IPv4Item(title='Interface') default_include = MinionDefaultInclude() include = IncludeConfig() diff --git a/salt/config/schemas/ssh.py b/salt/config/schemas/ssh.py index 485daccbca..0aa7327408 100644 --- a/salt/config/schemas/ssh.py +++ b/salt/config/schemas/ssh.py @@ -13,23 +13,23 @@ from __future__ import absolute_import # Import Salt libs -from salt.utils.config import (Configuration, - StringConfig, - IntegerConfig, - SecretConfig, - OneOfConfig, - IPv4Config, - HostnameConfig, - PortConfig, - BooleanConfig, +from salt.utils.schema import (Schema, + StringItem, + IntegerItem, + SecretItem, + OneOfItem, + IPv4Item, + HostnameItem, + PortItem, + BooleanItem, RequirementsItem, - DictConfig, - AnyOfConfig + DictItem, + AnyOfItem ) from salt.config.schemas.minion import MinionConfiguration -class RosterEntryConfig(Configuration): +class RosterEntryConfig(Schema): ''' Schema definition of a Salt SSH Roster entry ''' @@ -37,44 +37,44 @@ class RosterEntryConfig(Configuration): title = 'Roster Entry' description = 'Salt SSH roster entry definition' - host = StringConfig(title='Host', + host = StringItem(title='Host', description='The IP address or DNS name of the remote host', # Pretty naive pattern matching pattern=r'^((\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|([A-Za-z0-9][A-Za-z0-9\.\-]{1,255}))$', min_length=1, required=True) - port = PortConfig(title='Port', + port = PortItem(title='Port', description='The target system\'s ssh port number', default=22) - user = StringConfig(title='User', + user = StringItem(title='User', description='The user to log in as. Defaults to root', default='root', min_length=1, required=True) - passwd = SecretConfig(title='Password', + passwd = SecretItem(title='Password', description='The password to log in with') - priv = StringConfig(title='Private Key', + priv = StringItem(title='Private Key', description='File path to ssh private key, defaults to salt-ssh.rsa') - passwd_or_priv_requirement = AnyOfConfig(items=(RequirementsItem(requirements=['passwd']), + passwd_or_priv_requirement = AnyOfItem(items=(RequirementsItem(requirements=['passwd']), RequirementsItem(requirements=['priv'])))(flatten=True) - sudo = BooleanConfig(title='Sudo', + sudo = BooleanItem(title='Sudo', description='run command via sudo. Defaults to False', default=False) - timeout = IntegerConfig(title='Timeout', + timeout = IntegerItem(title='Timeout', description=('Number of seconds to wait for response ' 'when establishing an SSH connection')) - thin_dir = StringConfig(title='Thin Directory', + thin_dir = StringItem(title='Thin Directory', description=('The target system\'s storage directory for Salt ' 'components. Defaults to /tmp/salt-.')) - minion_opts = DictConfig(title='Minion Options', + minion_opts = DictItem(title='Minion Options', description='Dictionary of minion options', properties=MinionConfiguration()) -class RosterConfig(Configuration): +class RosterItem(Schema): title = 'Roster Configuration' description = 'Roster entries definition' - roster_entries = DictConfig( + roster_entries = DictItem( pattern_properties={ r'^([^:]+)$': RosterEntryConfig()})(flatten=True) From 08692b6cfcac2681f7c3f7006aa6a2017bca4573 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 24 Jul 2015 13:47:05 +0100 Subject: [PATCH 17/26] Indentation fixes --- salt/config/schemas/ssh.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/salt/config/schemas/ssh.py b/salt/config/schemas/ssh.py index 0aa7327408..81307e6519 100644 --- a/salt/config/schemas/ssh.py +++ b/salt/config/schemas/ssh.py @@ -38,28 +38,28 @@ class RosterEntryConfig(Schema): description = 'Salt SSH roster entry definition' host = StringItem(title='Host', - description='The IP address or DNS name of the remote host', - # Pretty naive pattern matching - pattern=r'^((\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|([A-Za-z0-9][A-Za-z0-9\.\-]{1,255}))$', - min_length=1, - required=True) + description='The IP address or DNS name of the remote host', + # Pretty naive pattern matching + pattern=r'^((\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|([A-Za-z0-9][A-Za-z0-9\.\-]{1,255}))$', + min_length=1, + required=True) port = PortItem(title='Port', - description='The target system\'s ssh port number', - default=22) + description='The target system\'s ssh port number', + default=22) user = StringItem(title='User', - description='The user to log in as. Defaults to root', - default='root', - min_length=1, - required=True) + description='The user to log in as. Defaults to root', + default='root', + min_length=1, + required=True) passwd = SecretItem(title='Password', - description='The password to log in with') + description='The password to log in with') priv = StringItem(title='Private Key', - description='File path to ssh private key, defaults to salt-ssh.rsa') + description='File path to ssh private key, defaults to salt-ssh.rsa') passwd_or_priv_requirement = AnyOfItem(items=(RequirementsItem(requirements=['passwd']), - RequirementsItem(requirements=['priv'])))(flatten=True) + RequirementsItem(requirements=['priv'])))(flatten=True) sudo = BooleanItem(title='Sudo', - description='run command via sudo. Defaults to False', - default=False) + description='run command via sudo. Defaults to False', + default=False) timeout = IntegerItem(title='Timeout', description=('Number of seconds to wait for response ' 'when establishing an SSH connection')) From 10c47647e376835cb81f30ec32e94935056af916 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 24 Jul 2015 13:48:19 +0100 Subject: [PATCH 18/26] Also rename the test module --- tests/unit/utils/{config_test.py => schema_test.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/unit/utils/{config_test.py => schema_test.py} (100%) diff --git a/tests/unit/utils/config_test.py b/tests/unit/utils/schema_test.py similarity index 100% rename from tests/unit/utils/config_test.py rename to tests/unit/utils/schema_test.py From e1d5bd6ecd5dc8b904f58ee6c1ec299e07493288 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 27 Jul 2015 17:54:26 +0100 Subject: [PATCH 19/26] Allow passing JID to a salt-ssh execution --- salt/client/ssh/__init__.py | 8 ++++---- salt/client/ssh/client.py | 4 ++-- salt/config/schemas/ssh.py | 6 ++++-- salt/utils/parsers.py | 11 +++++++++++ 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py index a890697e64..85156d3ec8 100644 --- a/salt/client/ssh/__init__.py +++ b/salt/client/ssh/__init__.py @@ -446,7 +446,7 @@ class SSH(object): if len(running) >= self.opts.get('ssh_max_procs', 25) or len(self.targets) >= len(running): time.sleep(0.1) - def run_iter(self, mine=False): + def run_iter(self, mine=False, jid=None): ''' Execute and yield returns as they come in, do not print to the display @@ -456,7 +456,7 @@ class SSH(object): will modify the argv with the arguments from mine_functions ''' fstr = '{0}.prep_jid'.format(self.opts['master_job_cache']) - jid = self.returners[fstr]() + jid = self.returners[fstr](passed_jid=jid or self.opts.get('jid', None)) # Save the invocation information argv = self.opts['argv'] @@ -500,12 +500,12 @@ class SSH(object): 'return': ret, 'fun': fun}) - def run(self): + def run(self, jid=None): ''' Execute the overall routine, print results via outputters ''' fstr = '{0}.prep_jid'.format(self.opts['master_job_cache']) - jid = self.returners[fstr]() + jid = self.returners[fstr](passed_jid=jid or self.opts.get('jid', None)) # Save the invocation information argv = self.opts['argv'] diff --git a/salt/client/ssh/client.py b/salt/client/ssh/client.py index f9ac069a36..447d6ae228 100644 --- a/salt/client/ssh/client.py +++ b/salt/client/ssh/client.py @@ -82,7 +82,7 @@ class SSHClient(object): expr_form, kwarg, **kwargs) - for ret in ssh.run_iter(): + for ret in ssh.run_iter(jid=kwargs.get('jid', None)): yield ret def cmd( @@ -109,7 +109,7 @@ class SSHClient(object): kwarg, **kwargs) final = {} - for ret in ssh.run_iter(): + for ret in ssh.run_iter(jid=kwargs.get('jid', None)): final.update(ret) return final diff --git a/salt/config/schemas/ssh.py b/salt/config/schemas/ssh.py index 81307e6519..a2824c457a 100644 --- a/salt/config/schemas/ssh.py +++ b/salt/config/schemas/ssh.py @@ -52,9 +52,11 @@ class RosterEntryConfig(Schema): min_length=1, required=True) passwd = SecretItem(title='Password', - description='The password to log in with') + description='The password to log in with', + min_length=1) priv = StringItem(title='Private Key', - description='File path to ssh private key, defaults to salt-ssh.rsa') + description='File path to ssh private key, defaults to salt-ssh.rsa', + min_length=1) passwd_or_priv_requirement = AnyOfItem(items=(RequirementsItem(requirements=['passwd']), RequirementsItem(requirements=['priv'])))(flatten=True) sudo = BooleanItem(title='Sudo', diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py index eabe877c1e..736da6bf5a 100644 --- a/salt/utils/parsers.py +++ b/salt/utils/parsers.py @@ -32,6 +32,7 @@ import salt.utils as utils import salt.version as version import salt.utils.args import salt.utils.xdg +import salt.utils.jid from salt.utils import kinds from salt.defaults import DEFAULT_TARGET_DELIM from salt.utils.validate.path import is_writeable @@ -2456,6 +2457,11 @@ class SaltSSHOptionParser(six.with_metaclass(OptionParserMeta, default='python3', help='Path to a python3 binary which has salt installed' ) + self.add_option( + '--jid', + default=None, + help='Pass a JID to be used instead of generating one' + ) auth_group = optparse.OptionGroup( self, 'Authentication Options', @@ -2566,6 +2572,11 @@ class SaltSSHOptionParser(six.with_metaclass(OptionParserMeta, def setup_config(self): return config.master_config(self.get_config_file_path()) + def process_jid(self): + if self.options.jid is not None: + if not salt.utils.jid.is_jid(self.options.jid): + self.error('\'{0}\' is not a valid JID'.format(self.options.jid)) + class SaltCloudParser(six.with_metaclass(OptionParserMeta, OptionParser, From 4faf2248cd96e20aa9afd66d95dee4182dc01f69 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 28 Jul 2015 00:54:58 +0100 Subject: [PATCH 20/26] Comment typo --- salt/utils/thin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/thin.py b/salt/utils/thin.py index b9dff07673..5219a1ed6d 100644 --- a/salt/utils/thin.py +++ b/salt/utils/thin.py @@ -185,7 +185,7 @@ def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='', else: tops_py_version_mapping['3'] = tops - # TODO: Consider putting know py2 and py3 compatible libs in it's own sharable directory. + # TODO: Consider putting known py2 and py3 compatible libs in it's own sharable directory. # This would reduce the thin size. if six.PY2 and sys.version_info[0] == 2: # Get python 3 tops From fcef605a4ac775f8fed64db6333f7c4483ecb3b7 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 28 Jul 2015 02:21:11 +0100 Subject: [PATCH 21/26] Remove unused imports --- salt/config/schemas/ssh.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/salt/config/schemas/ssh.py b/salt/config/schemas/ssh.py index a2824c457a..5b0262fa38 100644 --- a/salt/config/schemas/ssh.py +++ b/salt/config/schemas/ssh.py @@ -17,9 +17,6 @@ from salt.utils.schema import (Schema, StringItem, IntegerItem, SecretItem, - OneOfItem, - IPv4Item, - HostnameItem, PortItem, BooleanItem, RequirementsItem, From d6492528787dfc8d51062676997fbc64e2e9f219 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 28 Jul 2015 02:24:58 +0100 Subject: [PATCH 22/26] Fix class reference --- tests/unit/config/schemas/test_ssh.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/unit/config/schemas/test_ssh.py b/tests/unit/config/schemas/test_ssh.py index 68eb376067..ac833fe847 100644 --- a/tests/unit/config/schemas/test_ssh.py +++ b/tests/unit/config/schemas/test_ssh.py @@ -6,7 +6,7 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ''' # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function # Import Salt Testing Libs from salttesting import TestCase, skipIf @@ -17,7 +17,6 @@ ensure_in_syspath('../../') # Import Salt Libs from salt.config.schemas import ssh as ssh_schemas from salt.config.schemas.minion import MinionConfiguration -from salt.utils.config import DictConfig # Import 3rd-party libs try: @@ -88,9 +87,9 @@ class RoosterEntryConfigTest(TestCase): 'title': 'Thin Directory' }, # The actuall representation of the minion options would make this HUGE! - 'minion_opts': DictConfig(title='Minion Options', - description='Dictionary of minion options', - properties=MinionConfiguration()).serialize(), + 'minion_opts': ssh_schemas.DictItem(title='Minion Options', + description='Dictionary of minion options', + properties=MinionConfiguration()).serialize(), }, 'anyOf': [ { @@ -244,7 +243,7 @@ class RoosterEntryConfigTest(TestCase): self.assertIn('is not of type', excinfo.exception.message) -class RosterConfigTest(TestCase): +class RosterItemTest(TestCase): def test_roster_config(self): try: @@ -259,11 +258,11 @@ class RosterConfigTest(TestCase): }, "additionalProperties": False }, - ssh_schemas.RosterConfig.serialize() + ssh_schemas.RosterItem.serialize() ) except AssertionError: import json - print(json.dumps(ssh_schemas.RosterConfig.serialize(), indent=4)) + print(json.dumps(ssh_schemas.RosterItem.serialize(), indent=4)) raise @skipIf(HAS_JSONSCHEMA is False, 'The \'jsonschema\' library is missing') @@ -277,7 +276,7 @@ class RosterConfigTest(TestCase): 'passwd': 'foo' } }, - ssh_schemas.RosterConfig.serialize(), + ssh_schemas.RosterItem.serialize(), format_checker=jsonschema.FormatChecker() ) except jsonschema.exceptions.ValidationError as exc: @@ -292,7 +291,7 @@ class RosterConfigTest(TestCase): 'passwd': 'foo' } }, - ssh_schemas.RosterConfig.serialize(), + ssh_schemas.RosterItem.serialize(), format_checker=jsonschema.FormatChecker() ) self.assertIn( From e7681f7c7558504fdc3f1bb0b37503a23c4d1e9c Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 28 Jul 2015 12:07:04 +0100 Subject: [PATCH 23/26] Try to overcome a failure to get cwd in Cent tests --- tests/integration/__init__.py | 6 ++++++ tests/integration/shell/call.py | 2 +- tests/integration/shell/cp.py | 2 +- tests/integration/shell/key.py | 2 +- tests/integration/shell/master.py | 2 +- tests/integration/shell/matcher.py | 2 +- tests/integration/shell/minion.py | 2 +- tests/integration/shell/runner.py | 2 +- tests/integration/shell/syndic.py | 2 +- tests/runtests.py | 6 +++--- 10 files changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 05af24d433..2731471167 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1200,6 +1200,12 @@ class ShellCase(AdaptedConfigurationTestCaseMixIn, ShellTestCase): _script_dir_ = SCRIPT_DIR _python_executable_ = PYEXEC + def chdir(self, dirname): + try: + os.chdir(dirname) + except OSError: + os.chdir(INTEGRATION_TEST_DIR) + def run_salt(self, arg_str, with_retcode=False, catch_stderr=False): ''' Execute salt diff --git a/tests/integration/shell/call.py b/tests/integration/shell/call.py index 11fefaa692..f941039f43 100644 --- a/tests/integration/shell/call.py +++ b/tests/integration/shell/call.py @@ -297,7 +297,7 @@ class CallTest(integration.ShellCase, integration.ShellCaseCommonTestsMixIn): ) self.assertEqual(ret[2], 2) finally: - os.chdir(old_cwd) + self.chdir(old_cwd) if os.path.isdir(config_dir): shutil.rmtree(config_dir) diff --git a/tests/integration/shell/cp.py b/tests/integration/shell/cp.py index dc4b8860cd..12dfe4bf9e 100644 --- a/tests/integration/shell/cp.py +++ b/tests/integration/shell/cp.py @@ -159,7 +159,7 @@ class CopyTest(integration.ShellCase, integration.ShellCaseCommonTestsMixIn): self.assertEqual(ret[2], 2) finally: if old_cwd is not None: - os.chdir(old_cwd) + self.chdir(old_cwd) if os.path.isdir(config_dir): shutil.rmtree(config_dir) diff --git a/tests/integration/shell/key.py b/tests/integration/shell/key.py index e1dacf47d4..262a110385 100644 --- a/tests/integration/shell/key.py +++ b/tests/integration/shell/key.py @@ -254,7 +254,7 @@ class KeyTest(integration.ShellCase, integration.ShellCaseCommonTestsMixIn): self.assertIn('minion', '\n'.join(ret)) self.assertFalse(os.path.isdir(os.path.join(config_dir, 'file:'))) finally: - os.chdir(old_cwd) + self.chdir(old_cwd) if os.path.isdir(config_dir): shutil.rmtree(config_dir) diff --git a/tests/integration/shell/master.py b/tests/integration/shell/master.py index e059e7fbf3..b160e38fd4 100644 --- a/tests/integration/shell/master.py +++ b/tests/integration/shell/master.py @@ -74,7 +74,7 @@ class MasterTest(integration.ShellCase, integration.ShellCaseCommonTestsMixIn): ) self.assertEqual(ret[2], 2) finally: - os.chdir(old_cwd) + self.chdir(old_cwd) if os.path.isdir(config_dir): shutil.rmtree(config_dir) diff --git a/tests/integration/shell/matcher.py b/tests/integration/shell/matcher.py index e974d39d28..4bfc77cae3 100644 --- a/tests/integration/shell/matcher.py +++ b/tests/integration/shell/matcher.py @@ -345,7 +345,7 @@ class MatchTest(integration.ShellCase, integration.ShellCaseCommonTestsMixIn): ) self.assertEqual(ret[2], 2) finally: - os.chdir(old_cwd) + self.chdir(old_cwd) if os.path.isdir(config_dir): shutil.rmtree(config_dir) diff --git a/tests/integration/shell/minion.py b/tests/integration/shell/minion.py index 498a11102d..38e3cbba24 100644 --- a/tests/integration/shell/minion.py +++ b/tests/integration/shell/minion.py @@ -71,7 +71,7 @@ class MinionTest(integration.ShellCase, integration.ShellCaseCommonTestsMixIn): ) self.assertEqual(ret[2], 2) finally: - os.chdir(old_cwd) + self.chdir(old_cwd) if os.path.isdir(config_dir): shutil.rmtree(config_dir) diff --git a/tests/integration/shell/runner.py b/tests/integration/shell/runner.py index b2399f8c69..bcb3c985cc 100644 --- a/tests/integration/shell/runner.py +++ b/tests/integration/shell/runner.py @@ -94,7 +94,7 @@ class RunTest(integration.ShellCase, integration.ShellCaseCommonTestsMixIn): ) self.assertEqual(ret[2], 2) finally: - os.chdir(old_cwd) + self.chdir(old_cwd) if os.path.isdir(config_dir): shutil.rmtree(config_dir) diff --git a/tests/integration/shell/syndic.py b/tests/integration/shell/syndic.py index ed24301627..6c0e9cc818 100644 --- a/tests/integration/shell/syndic.py +++ b/tests/integration/shell/syndic.py @@ -75,7 +75,7 @@ class SyndicTest(integration.ShellCase, integration.ShellCaseCommonTestsMixIn): ) self.assertEqual(ret[2], 2) finally: - os.chdir(old_cwd) + self.chdir(old_cwd) if os.path.isdir(config_dir): shutil.rmtree(config_dir) diff --git a/tests/runtests.py b/tests/runtests.py index 6bee360b2b..a107ad5506 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -14,13 +14,13 @@ import time # Import salt libs from integration import TestDaemon, TMP # pylint: disable=W0403 +from integration import INTEGRATION_TEST_DIR +from integration import CODE_DIR as SALT_ROOT # Import Salt Testing libs from salttesting.parser import PNUM, print_header from salttesting.parser.cover import SaltCoverageTestingParser -TEST_DIR = os.path.dirname(os.path.normpath(os.path.abspath(__file__))) -SALT_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) XML_OUTPUT_DIR = os.environ.get( 'SALT_XML_TEST_REPORTS_DIR', os.path.join(TMP, 'xml-test-reports') @@ -30,7 +30,7 @@ HTML_OUTPUT_DIR = os.environ.get( os.path.join(TMP, 'html-test-reports') ) - +TEST_DIR = os.path.dirname(INTEGRATION_TEST_DIR) try: if SALT_ROOT: os.chdir(SALT_ROOT) From c7b5fccb9e54dfb91607be3f8f3bd09491269ab8 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 28 Jul 2015 12:22:53 +0100 Subject: [PATCH 24/26] Change cwd back to Salt's code dir --- tests/unit/utils/cloud_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/utils/cloud_test.py b/tests/unit/utils/cloud_test.py index 825d90afa5..1bdd26e08f 100644 --- a/tests/unit/utils/cloud_test.py +++ b/tests/unit/utils/cloud_test.py @@ -20,7 +20,7 @@ ensure_in_syspath('../../') # Import salt libs from salt.utils import cloud -from integration import TMP +from integration import TMP, CODE_DIR GPG_KEYDIR = os.path.join(TMP, 'gpg-keydir') @@ -63,6 +63,8 @@ try: except ImportError: HAS_KEYRING = False +os.chdir(CODE_DIR) + class CloudUtilsTestCase(TestCase): From 034353040c0008eeb0164efad3d24ccdde7de6bb Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 28 Jul 2015 12:23:18 +0100 Subject: [PATCH 25/26] Make the tests discoverable --- tests/unit/config/schemas/{test_ssh.py => ssh_test.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/unit/config/schemas/{test_ssh.py => ssh_test.py} (100%) diff --git a/tests/unit/config/schemas/test_ssh.py b/tests/unit/config/schemas/ssh_test.py similarity index 100% rename from tests/unit/config/schemas/test_ssh.py rename to tests/unit/config/schemas/ssh_test.py From 359348ba1c74db5bc3fb39df6e4e5a327bea935f Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 28 Jul 2015 12:37:19 +0100 Subject: [PATCH 26/26] Fix SSH Roster Schema unittests --- tests/unit/config/schemas/ssh_test.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/unit/config/schemas/ssh_test.py b/tests/unit/config/schemas/ssh_test.py index ac833fe847..02b47ef7cc 100644 --- a/tests/unit/config/schemas/ssh_test.py +++ b/tests/unit/config/schemas/ssh_test.py @@ -63,12 +63,14 @@ class RoosterEntryConfigTest(TestCase): 'title': 'Password', 'type': 'string', 'description': 'The password to log in with', - 'format': 'secret' + 'format': 'secret', + 'minLength': 1 }, 'priv': { 'type': 'string', 'description': 'File path to ssh private key, defaults to salt-ssh.rsa', - 'title': 'Private Key' + 'title': 'Private Key', + 'minLength': 1 }, 'sudo': { 'default': False, @@ -247,10 +249,10 @@ class RosterItemTest(TestCase): def test_roster_config(self): try: - self.assertDictEqual( + self.assertDictContainsSubset( { "$schema": "http://json-schema.org/draft-04/schema#", - "title": "roster_entries", + "title": "Roster Configuration", "description": "Roster entries definition", "type": "object", "patternProperties": {