From 553aaad105434d25c982a90b16b8387163d89ffa Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Sun, 23 Apr 2017 23:36:37 -0600 Subject: [PATCH 001/184] Add get_jids_filter method to pgjsonb --- salt/returners/pgjsonb.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/salt/returners/pgjsonb.py b/salt/returners/pgjsonb.py index 126b29abae..65bd162375 100644 --- a/salt/returners/pgjsonb.py +++ b/salt/returners/pgjsonb.py @@ -382,6 +382,30 @@ def get_jids(): return ret +def get_jids_filter(count, filter_find_job=True): + ''' + Return a list of all job ids + :param int count: show not more than the count of most recent jobs + :param bool filter_find_jobs: filter out 'saltutil.find_job' jobs + ''' + with _get_serv(ret=None, commit=True) as cur: + + sql = '''SELECT * FROM ( + SELECT DISTINCT jid ,load FROM jids + {0} + ORDER BY jid DESC limit {1} + ) tmp + ORDER BY jid;''' + where = '''WHERE 'load' NOT LIKE '%"fun": "saltutil.find_job"%' ''' + + cur.execute(sql.format(where if filter_find_job else '', count)) + data = cur.fetchall() + ret = [] + for jid, load in data: + ret.append(salt.utils.jid.format_jid_instance_ext(jid,load)) + return ret + + def get_minions(): ''' Return a list of minions From d26d961fb09229c5beec181a8be7f1585196cb4e Mon Sep 17 00:00:00 2001 From: Kunal Ajay Bajpai Date: Tue, 12 Sep 2017 13:33:41 +0530 Subject: [PATCH 002/184] Fix check for returner save_load --- salt/utils/job.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/utils/job.py b/salt/utils/job.py index c37e034c32..8fe2167612 100644 --- a/salt/utils/job.py +++ b/salt/utils/job.py @@ -100,8 +100,7 @@ def store_job(opts, load, event=None, mminion=None): raise KeyError(emsg) if 'jid' in load \ - and 'get_load' in mminion.returners \ - and not mminion.returners[getfstr](load.get('jid', '')): + and getfstr in mminion.returners: mminion.returners[savefstr](load['jid'], load) mminion.returners[fstr](load) From 847710c4331399dcc54cedd127dbe5f17af7d9e2 Mon Sep 17 00:00:00 2001 From: Kunal Ajay Bajpai Date: Wed, 13 Sep 2017 11:55:09 +0530 Subject: [PATCH 003/184] Remove redundant check and add try/except --- salt/utils/job.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/utils/job.py b/salt/utils/job.py index 8fe2167612..ad40213848 100644 --- a/salt/utils/job.py +++ b/salt/utils/job.py @@ -99,9 +99,10 @@ def store_job(opts, load, event=None, mminion=None): log.error(emsg) raise KeyError(emsg) - if 'jid' in load \ - and getfstr in mminion.returners: + try: mminion.returners[savefstr](load['jid'], load) + except KeyError as e: + log.error("Load does not contain 'jid': %s", e) mminion.returners[fstr](load) if (opts.get('job_cache_store_endtime') From ad80801d0f9ef764d6fb179e97225e5865a5a767 Mon Sep 17 00:00:00 2001 From: Alexei Pope Lane Date: Wed, 13 Sep 2017 14:54:10 +1000 Subject: [PATCH 004/184] Improvements to SSH known_hosts module and state. --- salt/modules/ssh.py | 231 +++++++++++++++++----- salt/states/ssh_known_hosts.py | 6 +- tests/integration/modules/test_ssh.py | 20 +- tests/integration/states/test_ssh.py | 14 +- tests/support/case.py | 2 +- tests/unit/states/test_ssh_known_hosts.py | 10 +- 6 files changed, 204 insertions(+), 79 deletions(-) diff --git a/salt/modules/ssh.py b/salt/modules/ssh.py index cba2035534..4dbf0f4c2c 100644 --- a/salt/modules/ssh.py +++ b/salt/modules/ssh.py @@ -795,6 +795,22 @@ def set_auth_key( return 'new' +def _get_matched_host_line_numbers(lines, enc): + ''' + Helper function which parses ssh-keygen -F function output and yield line + number of known_hosts entries with encryption key type matching enc, + one by one. + ''' + enc = enc if enc else "rsa" + for i, line in enumerate(lines): + if i % 2 == 0: + line_no = int(line.strip().split()[-1]) + line_enc = lines[i + 1].strip().split()[-2] + if line_enc != enc: + continue + yield line_no + + def _parse_openssh_output(lines, fingerprint_hash_type=None): ''' Helper function which parses ssh-keygen -F and ssh-keyscan function output @@ -837,6 +853,33 @@ def get_known_host(user, salt '*' ssh.get_known_host ''' + salt.utils.warn_until( + 'Neon', + 'get_known_host has been deprecated in favour of ' + 'get_known_host_entries.' + ) + known_hosts = get_known_host_entries(user, hostname, config, port, fingerprint_hash_type) + return known_hosts[0] if known_hosts else None + + +@salt.utils.decorators.path.which('ssh-keygen') +def get_known_host_entries(user, + hostname, + config=None, + port=None, + fingerprint_hash_type=None): + ''' + .. versionadded:: Oxygen + + Return information about known host entries from the configfile, if any. + If there are no entries for a matching hostname, return None. + + CLI Example: + + .. code-block:: bash + + salt '*' ssh.get_known_host_entries + ''' full = _get_known_hosts_file(config=config, user=user) if isinstance(full, dict): @@ -847,11 +890,11 @@ def get_known_host(user, lines = __salt__['cmd.run'](cmd, ignore_retcode=True, python_shell=False).splitlines() - known_hosts = list( + known_host_entries = list( _parse_openssh_output(lines, fingerprint_hash_type=fingerprint_hash_type) ) - return known_hosts[0] if known_hosts else None + return known_host_entries if known_host_entries else None @salt.utils.decorators.path.which('ssh-keyscan') @@ -872,9 +915,8 @@ def recv_known_host(hostname, or ssh-dss port - optional parameter, denoting the port of the remote host, which will be - used in case, if the public key will be requested from it. By default - the port 22 is used. + Optional parameter, denoting the port of the remote host on which an + SSH daemon is running. By default the port 22 is used. hash_known_hosts : True Hash all hostnames and addresses in the known hosts file. @@ -888,8 +930,8 @@ def recv_known_host(hostname, .. versionadded:: 2016.3.0 fingerprint_hash_type - The public key fingerprint hash type that the public key fingerprint - was originally hashed with. This defaults to ``sha256`` if not specified. + The fingerprint hash type that the public key fingerprints were + originally hashed with. This defaults to ``sha256`` if not specified. .. versionadded:: 2016.11.4 .. versionchanged:: 2017.7.0: default changed from ``md5`` to ``sha256`` @@ -900,6 +942,60 @@ def recv_known_host(hostname, salt '*' ssh.recv_known_host enc= port= ''' + salt.utils.warn_until( + 'Neon', + 'recv_known_host has been deprecated in favour of ' + 'recv_known_host_entries.' + ) + known_hosts = recv_known_host_entries(hostname, enc, port, hash_known_hosts, timeout, fingerprint_hash_type) + return known_hosts[0] if known_hosts else None + + +@salt.utils.decorators.path.which('ssh-keyscan') +def recv_known_host_entries(hostname, + enc=None, + port=None, + hash_known_hosts=True, + timeout=5, + fingerprint_hash_type=None): + ''' + .. versionadded:: Oxygen + + Retrieve information about host public keys from remote server + + hostname + The name of the remote host (e.g. "github.com") + + enc + Defines what type of key is being used, can be ed25519, ecdsa ssh-rsa + or ssh-dss + + port + Optional parameter, denoting the port of the remote host on which an + SSH daemon is running. By default the port 22 is used. + + hash_known_hosts : True + Hash all hostnames and addresses in the known hosts file. + + timeout : int + Set the timeout for connection attempts. If ``timeout`` seconds have + elapsed since a connection was initiated to a host or since the last + time anything was read from that host, then the connection is closed + and the host in question considered unavailable. Default is 5 seconds. + + fingerprint_hash_type + The fingerprint hash type that the public key fingerprints were + originally hashed with. This defaults to ``sha256`` if not specified. + + .. versionadded:: 2016.11.4 + .. versionchanged:: 2017.7.0: default changed from ``md5`` to ``sha256`` + + CLI Example: + + .. code-block:: bash + + salt '*' ssh.recv_known_host_entries enc= port= + ''' # The following list of OSes have an old version of openssh-clients # and thus require the '-t' option for ssh-keyscan need_dash_t = ('CentOS-5',) @@ -920,9 +1016,9 @@ def recv_known_host(hostname, while not lines and attempts > 0: attempts = attempts - 1 lines = __salt__['cmd.run'](cmd, python_shell=False).splitlines() - known_hosts = list(_parse_openssh_output(lines, + known_host_entries = list(_parse_openssh_output(lines, fingerprint_hash_type=fingerprint_hash_type)) - return known_hosts[0] if known_hosts else None + return known_host_entries if known_host_entries else None def check_known_host(user=None, hostname=None, key=None, fingerprint=None, @@ -953,18 +1049,20 @@ def check_known_host(user=None, hostname=None, key=None, fingerprint=None, else: config = config or '.ssh/known_hosts' - known_host = get_known_host(user, + known_host_entries = get_known_host_entries(user, hostname, config=config, port=port, fingerprint_hash_type=fingerprint_hash_type) + known_keys = [h['key'] for h in known_host_entries] if known_host_entries else [] + known_fingerprints = [h['fingerprint'] for h in known_host_entries] if known_host_entries else [] - if not known_host or 'fingerprint' not in known_host: + if not known_host_entries: return 'add' if key: - return 'exists' if key == known_host['key'] else 'update' + return 'exists' if key in known_keys else 'update' elif fingerprint: - return ('exists' if fingerprint == known_host['fingerprint'] + return ('exists' if fingerprint in known_fingerprints else 'update') else: return 'exists' @@ -1084,70 +1182,99 @@ def set_known_host(user=None, update_required = False check_required = False - stored_host = get_known_host(user, + stored_host_entries = get_known_host_entries(user, hostname, config=config, port=port, fingerprint_hash_type=fingerprint_hash_type) + stored_keys = [h['key'] for h in stored_host_entries] if stored_host_entries else [] + stored_fingerprints = [h['fingerprint'] for h in stored_host_entries] if stored_host_entries else [] - if not stored_host: + if not stored_host_entries: update_required = True - elif fingerprint and fingerprint != stored_host['fingerprint']: + elif fingerprint and fingerprint not in stored_fingerprints: update_required = True - elif key and key != stored_host['key']: + elif key and key not in stored_keys: update_required = True - elif key != stored_host['key']: + elif key is None and fingerprint is None: check_required = True if not update_required and not check_required: - return {'status': 'exists', 'key': stored_host['key']} + return {'status': 'exists', 'keys': stored_keys} if not key: - remote_host = recv_known_host(hostname, + remote_host_entries = recv_known_host_entries(hostname, enc=enc, port=port, hash_known_hosts=hash_known_hosts, timeout=timeout, fingerprint_hash_type=fingerprint_hash_type) - if not remote_host: + known_keys = [h['key'] for h in remote_host_entries] if remote_host_entries else [] + known_fingerprints = [h['fingerprint'] for h in remote_host_entries] if remote_host_entries else [] + if not remote_host_entries: return {'status': 'error', - 'error': 'Unable to receive remote host key'} + 'error': 'Unable to receive remote host keys'} - if fingerprint and fingerprint != remote_host['fingerprint']: + if fingerprint and fingerprint not in known_fingerprints: return {'status': 'error', - 'error': ('Remote host public key found but its fingerprint ' - 'does not match one you have provided')} + 'error': ('Remote host public keys found but none of their' + 'fingerprints match the one you have provided')} if check_required: - if remote_host['key'] == stored_host['key']: - return {'status': 'exists', 'key': stored_host['key']} + for key in known_keys: + if key in stored_keys: + return {'status': 'exists', 'keys': stored_keys} full = _get_known_hosts_file(config=config, user=user) if isinstance(full, dict): return full - # Get information about the known_hosts file before rm_known_host() - # because it will create a new file with mode 0600 - orig_known_hosts_st = None - try: - orig_known_hosts_st = os.stat(full) - except OSError as exc: - if exc.args[1] == 'No such file or directory': - log.debug('{0} doesnt exist. Nothing to preserve.'.format(full)) + if os.path.isfile(full): + origmode = os.stat(full).st_mode - # remove everything we had in the config so far - rm_known_host(user, hostname, config=config) + # remove existing known_host entry with matching hostname and encryption key type + # use ssh-keygen -F to find the specific line(s) for this host + enc combo + ssh_hostname = _hostname_and_port_to_ssh_hostname(hostname, port) + cmd = ['ssh-keygen', '-F', ssh_hostname, '-f', full] + lines = __salt__['cmd.run'](cmd, + ignore_retcode=True, + python_shell=False).splitlines() + remove_lines = list( + _get_matched_host_line_numbers(lines, enc) + ) + + if remove_lines: + try: + with salt.utils.files.fopen(full, 'r+') as ofile: + known_hosts_lines = list(ofile) + # Delete from last line to first to avoid invalidating earlier indexes + for line_no in sorted(remove_lines, reverse=True): + del known_hosts_lines[line_no - 1] + # Write out changed known_hosts file + ofile.seek(0) + ofile.truncate() + for line in known_hosts_lines: + ofile.write(line) + except (IOError, OSError) as exception: + raise CommandExecutionError( + "Couldn't remove old entry(ies) from known hosts file: '{0}'".format(exception) + ) + else: + origmode = None # set up new value if key: - remote_host = {'hostname': hostname, 'enc': enc, 'key': key} + remote_host_entries = [{'hostname': hostname, 'enc': enc, 'key': key}] - if hash_known_hosts or port in [DEFAULT_SSH_PORT, None] or ':' in remote_host['hostname']: - line = '{hostname} {enc} {key}\n'.format(**remote_host) - else: - remote_host['port'] = port - line = '[{hostname}]:{port} {enc} {key}\n'.format(**remote_host) + lines = [] + for entry in remote_host_entries: + if hash_known_hosts or port in [DEFAULT_SSH_PORT, None] or ':' in entry['hostname']: + line = '{hostname} {enc} {key}\n'.format(**entry) + else: + entry['port'] = port + line = '[{hostname}]:{port} {enc} {key}\n'.format(**entry) + lines.append(line) # ensure ~/.ssh exists ssh_dir = os.path.dirname(full) @@ -1173,27 +1300,25 @@ def set_known_host(user=None, # write line to known_hosts file try: with salt.utils.files.fopen(full, 'a') as ofile: - ofile.write(line) + for line in lines: + ofile.write(line) except (IOError, OSError) as exception: raise CommandExecutionError( "Couldn't append to known hosts file: '{0}'".format(exception) ) - if os.geteuid() == 0: - if user: - os.chown(full, uinfo['uid'], uinfo['gid']) - elif orig_known_hosts_st: - os.chown(full, orig_known_hosts_st.st_uid, orig_known_hosts_st.st_gid) - - if orig_known_hosts_st: - os.chmod(full, orig_known_hosts_st.st_mode) + if os.geteuid() == 0 and user: + os.chown(full, uinfo['uid'], uinfo['gid']) + if origmode: + os.chmod(full, origmode) else: os.chmod(full, 0o600) if key and hash_known_hosts: cmd_result = __salt__['ssh.hash_known_hosts'](user=user, config=full) - return {'status': 'updated', 'old': stored_host, 'new': remote_host} + rval = {'status': 'updated', 'old': stored_host_entries, 'new': remote_host_entries} + return rval def user_keys(user=None, pubfile=None, prvfile=None): diff --git a/salt/states/ssh_known_hosts.py b/salt/states/ssh_known_hosts.py index 93325aa18a..c19afdc14a 100644 --- a/salt/states/ssh_known_hosts.py +++ b/salt/states/ssh_known_hosts.py @@ -178,13 +178,13 @@ def present( return dict(ret, result=False, comment=result['error']) else: # 'updated' if key: - new_key = result['new']['key'] + new_key = result['new'][0]['key'] return dict(ret, changes={'old': result['old'], 'new': result['new']}, comment='{0}\'s key saved to {1} (key: {2})'.format( name, config, new_key)) else: - fingerprint = result['new']['fingerprint'] + fingerprint = result['new'][0]['fingerprint'] return dict(ret, changes={'old': result['old'], 'new': result['new']}, comment='{0}\'s key saved to {1} (fingerprint: {2})'.format( @@ -225,7 +225,7 @@ def absent(name, user=None, config=None): ret['result'] = False return dict(ret, comment=comment) - known_host = __salt__['ssh.get_known_host'](user=user, hostname=name, config=config) + known_host = __salt__['ssh.get_known_host_entries'](user=user, hostname=name, config=config) if not known_host: return dict(ret, comment='Host is already absent') diff --git a/tests/integration/modules/test_ssh.py b/tests/integration/modules/test_ssh.py index b565129ef0..6971e9921c 100644 --- a/tests/integration/modules/test_ssh.py +++ b/tests/integration/modules/test_ssh.py @@ -104,7 +104,7 @@ class SSHModuleTest(ModuleCase): # user will get an indicator of what went wrong. self.assertEqual(len(list(ret.items())), 0) # Zero keys found - def test_get_known_host(self): + def test_get_known_host_entries(self): ''' Check that known host information is returned from ~/.ssh/config ''' @@ -113,7 +113,7 @@ class SSHModuleTest(ModuleCase): KNOWN_HOSTS) arg = ['root', 'github.com'] kwargs = {'config': KNOWN_HOSTS} - ret = self.run_function('ssh.get_known_host', arg, **kwargs) + ret = self.run_function('ssh.get_known_host_entries', arg, **kwargs)[0] try: self.assertEqual(ret['enc'], 'ssh-rsa') self.assertEqual(ret['key'], self.key) @@ -125,16 +125,16 @@ class SSHModuleTest(ModuleCase): ) ) - def test_recv_known_host(self): + def test_recv_known_host_entries(self): ''' Check that known host information is returned from remote host ''' - ret = self.run_function('ssh.recv_known_host', ['github.com']) + ret = self.run_function('ssh.recv_known_host_entries', ['github.com']) try: self.assertNotEqual(ret, None) - self.assertEqual(ret['enc'], 'ssh-rsa') - self.assertEqual(ret['key'], self.key) - self.assertEqual(ret['fingerprint'], GITHUB_FINGERPRINT) + self.assertEqual(ret[0]['enc'], 'ssh-rsa') + self.assertEqual(ret[0]['key'], self.key) + self.assertEqual(ret[0]['fingerprint'], GITHUB_FINGERPRINT) except AssertionError as exc: raise AssertionError( 'AssertionError: {0}. Function returned: {1}'.format( @@ -215,7 +215,7 @@ class SSHModuleTest(ModuleCase): try: self.assertEqual(ret['status'], 'updated') self.assertEqual(ret['old'], None) - self.assertEqual(ret['new']['fingerprint'], GITHUB_FINGERPRINT) + self.assertEqual(ret['new'][0]['fingerprint'], GITHUB_FINGERPRINT) except AssertionError as exc: raise AssertionError( 'AssertionError: {0}. Function returned: {1}'.format( @@ -223,8 +223,8 @@ class SSHModuleTest(ModuleCase): ) ) # check that item does exist - ret = self.run_function('ssh.get_known_host', ['root', 'github.com'], - config=KNOWN_HOSTS) + ret = self.run_function('ssh.get_known_host_entries', ['root', 'github.com'], + config=KNOWN_HOSTS)[0] try: self.assertEqual(ret['fingerprint'], GITHUB_FINGERPRINT) except AssertionError as exc: diff --git a/tests/integration/states/test_ssh.py b/tests/integration/states/test_ssh.py index e09c6ad664..9f9372afcc 100644 --- a/tests/integration/states/test_ssh.py +++ b/tests/integration/states/test_ssh.py @@ -66,7 +66,7 @@ class SSHKnownHostsStateTest(ModuleCase, SaltReturnAssertsMixin): raise err self.assertSaltStateChangesEqual( - ret, GITHUB_FINGERPRINT, keys=('new', 'fingerprint') + ret, GITHUB_FINGERPRINT, keys=('new', 0, 'fingerprint') ) # save twice, no changes @@ -81,7 +81,7 @@ class SSHKnownHostsStateTest(ModuleCase, SaltReturnAssertsMixin): **dict(kwargs, name=GITHUB_IP)) try: self.assertSaltStateChangesEqual( - ret, GITHUB_FINGERPRINT, keys=('new', 'fingerprint') + ret, GITHUB_FINGERPRINT, keys=('new', 0, 'fingerprint') ) except AssertionError as err: try: @@ -94,8 +94,8 @@ class SSHKnownHostsStateTest(ModuleCase, SaltReturnAssertsMixin): # record for every host must be available ret = self.run_function( - 'ssh.get_known_host', ['root', 'github.com'], config=KNOWN_HOSTS - ) + 'ssh.get_known_host_entries', ['root', 'github.com'], config=KNOWN_HOSTS + )[0] try: self.assertNotIn(ret, ('', None)) except AssertionError: @@ -103,8 +103,8 @@ class SSHKnownHostsStateTest(ModuleCase, SaltReturnAssertsMixin): 'Salt return \'{0}\' is in (\'\', None).'.format(ret) ) ret = self.run_function( - 'ssh.get_known_host', ['root', GITHUB_IP], config=KNOWN_HOSTS - ) + 'ssh.get_known_host_entries', ['root', GITHUB_IP], config=KNOWN_HOSTS + )[0] try: self.assertNotIn(ret, ('', None, {})) except AssertionError: @@ -144,7 +144,7 @@ class SSHKnownHostsStateTest(ModuleCase, SaltReturnAssertsMixin): # remove once, the key is gone ret = self.run_state('ssh_known_hosts.absent', **kwargs) self.assertSaltStateChangesEqual( - ret, GITHUB_FINGERPRINT, keys=('old', 'fingerprint') + ret, GITHUB_FINGERPRINT, keys=('old', 0, 'fingerprint') ) # remove twice, nothing has changed diff --git a/tests/support/case.py b/tests/support/case.py index 1a757f5350..bb690f8a51 100644 --- a/tests/support/case.py +++ b/tests/support/case.py @@ -572,7 +572,7 @@ class ModuleCase(TestCase, SaltClientTestCaseMixin): behavior of the raw function call ''' know_to_return_none = ( - 'file.chown', 'file.chgrp', 'ssh.recv_known_host' + 'file.chown', 'file.chgrp', 'ssh.recv_known_host_entries' ) if 'f_arg' in kwargs: kwargs['arg'] = kwargs.pop('f_arg') diff --git a/tests/unit/states/test_ssh_known_hosts.py b/tests/unit/states/test_ssh_known_hosts.py index 88bee8963c..e74a7aebdb 100644 --- a/tests/unit/states/test_ssh_known_hosts.py +++ b/tests/unit/states/test_ssh_known_hosts.py @@ -96,7 +96,7 @@ class SshKnownHostsTestCase(TestCase, LoaderModuleMockMixin): self.assertDictEqual(ssh_known_hosts.present(name, user), ret) result = {'status': 'updated', 'error': '', - 'new': {'fingerprint': fingerprint, 'key': key}, + 'new': [{'fingerprint': fingerprint, 'key': key}], 'old': ''} mock = MagicMock(return_value=result) with patch.dict(ssh_known_hosts.__salt__, @@ -104,8 +104,8 @@ class SshKnownHostsTestCase(TestCase, LoaderModuleMockMixin): comt = ("{0}'s key saved to .ssh/known_hosts (key: {1})" .format(name, key)) ret.update({'comment': comt, 'result': True, - 'changes': {'new': {'fingerprint': fingerprint, - 'key': key}, 'old': ''}}) + 'changes': {'new': [{'fingerprint': fingerprint, + 'key': key}], 'old': ''}}) self.assertDictEqual(ssh_known_hosts.present(name, user, key=key), ret) @@ -136,14 +136,14 @@ class SshKnownHostsTestCase(TestCase, LoaderModuleMockMixin): mock = MagicMock(return_value=False) with patch.dict(ssh_known_hosts.__salt__, - {'ssh.get_known_host': mock}): + {'ssh.get_known_host_entries': mock}): comt = ('Host is already absent') ret.update({'comment': comt, 'result': True}) self.assertDictEqual(ssh_known_hosts.absent(name, user), ret) mock = MagicMock(return_value=True) with patch.dict(ssh_known_hosts.__salt__, - {'ssh.get_known_host': mock}): + {'ssh.get_known_host_entries': mock}): with patch.dict(ssh_known_hosts.__opts__, {'test': True}): comt = ('Key for github.com is set to be' ' removed from .ssh/known_hosts') From e0d20573ea43651b75315b6e23f30a5a9b397391 Mon Sep 17 00:00:00 2001 From: spenceation Date: Tue, 26 Sep 2017 14:33:02 -0400 Subject: [PATCH 005/184] - Added PANOS state modules for address and service object management. - Changed PANOS state module to only make state change if a change is required. - Added function to xmlutil to parse full XML data with attributes. The previous behavior was to ignore XML attributes. I previously added support in pull request 43574. However, this might cause unexpected behavior with modules dependent on this function and not currently utilizing XML attributes. (More details in previous behavior) - Added filters to pull dirtyId tags from candidate configuration queries. This tagging marks elements as modified and is primarily an internal system tag. This causes issues with queries and parsing. --- salt/modules/panos.py | 214 ++++++++++++++- salt/proxy/panos.py | 48 +++- salt/states/panos.py | 605 +++++++++++++++++++++++++++++++++++++++--- salt/utils/xmlutil.py | 92 +++++-- 4 files changed, 872 insertions(+), 87 deletions(-) diff --git a/salt/modules/panos.py b/salt/modules/panos.py index aecf93fffe..7a02a9decc 100644 --- a/salt/modules/panos.py +++ b/salt/modules/panos.py @@ -64,10 +64,10 @@ def _get_job_results(query=None): response = __proxy__['panos.call'](query) # If the response contains a job, we will wait for the results - if 'job' in response: - jid = response['job'] + if 'result' in response and 'job' in response['result']: + jid = response['result']['job'] - while get_job(jid)['job']['status'] != 'FIN': + while get_job(jid)['result']['job']['status'] != 'FIN': time.sleep(5) return get_job(jid) @@ -317,6 +317,56 @@ def fetch_license(auth_code=None): return __proxy__['panos.call'](query) +def get_address(address=None, vsys='1'): + ''' + Get the candidate configuration for the specified get_address object. This will not return address objects that are + marked as pre-defined objects. + + address(str): The name of the address object. + + vsys(str): The string representation of the VSYS ID. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_address myhost + salt '*' panos.get_address myhost 3 + + ''' + query = {'type': 'config', + 'action': 'get', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/vsys/entry[@name=\'vsys{0}\']/' + 'address/entry[@name=\'{1}\']'.format(vsys, address)} + + return __proxy__['panos.call'](query) + + +def get_address_group(addressgroup=None, vsys='1'): + ''' + Get the candidate configuration for the specified address group. This will not return address groups that are + marked as pre-defined objects. + + addressgroup(str): The name of the address group. + + vsys(str): The string representation of the VSYS ID. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_address_group foobar + salt '*' panos.get_address_group foobar 3 + + ''' + query = {'type': 'config', + 'action': 'get', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/vsys/entry[@name=\'vsys{0}\']/' + 'address-group/entry[@name=\'{1}\']'.format(vsys, addressgroup)} + + return __proxy__['panos.call'](query) + + def get_admins_active(): ''' Show active administrators. @@ -584,7 +634,7 @@ def get_hostname(): 'action': 'get', 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/hostname'} - return __proxy__['panos.call'](query)['hostname'] + return __proxy__['panos.call'](query) def get_interface_counters(name='all'): @@ -926,9 +976,30 @@ def get_platform(): return __proxy__['panos.call'](query) +def get_predefined_application(application=None): + ''' + Get the configuration for the specified pre-defined application object. This will only return pre-defined + application objects. + + application(str): The name of the pre-defined application object. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_predefined_application saltstack + + ''' + query = {'type': 'config', + 'action': 'get', + 'xpath': '/config/predefined/application/entry[@name=\'{0}\']'.format(application)} + + return __proxy__['panos.call'](query) + + def get_security_rule(rulename=None, vsys='1'): ''' - Get the candidate configuration for the specified rule. + Get the candidate configuration for the specified security rule. rulename(str): The name of the security rule. @@ -950,6 +1021,56 @@ def get_security_rule(rulename=None, vsys='1'): return __proxy__['panos.call'](query) +def get_service(service=None, vsys='1'): + ''' + Get the candidate configuration for the specified service object. This will not return services that are marked + as pre-defined objects. + + service(str): The name of the service object. + + vsys(str): The string representation of the VSYS ID. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_service tcp-443 + salt '*' panos.get_service tcp-443 3 + + ''' + query = {'type': 'config', + 'action': 'get', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/vsys/entry[@name=\'vsys{0}\']/' + 'service/entry[@name=\'{1}\']'.format(vsys, service)} + + return __proxy__['panos.call'](query) + + +def get_service_group(servicegroup=None, vsys='1'): + ''' + Get the candidate configuration for the specified service group. This will not return service groups that are + marked as pre-defined objects. + + servicegroup(str): The name of the service group. + + vsys(str): The string representation of the VSYS ID. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_service_group foobar + salt '*' panos.get_service_group foobar 3 + + ''' + query = {'type': 'config', + 'action': 'get', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/vsys/entry[@name=\'vsys{0}\']/' + 'service-group/entry[@name=\'{1}\']'.format(vsys, servicegroup)} + + return __proxy__['panos.call'](query) + + def get_session_info(): ''' Show device session statistics. @@ -1065,11 +1186,11 @@ def get_system_services(): return __proxy__['panos.call'](query) -def get_system_state(filter=None): +def get_system_state(mask=None): ''' Show the system state variables. - filter + mask Filters by a subtree or a wildcard. CLI Example: @@ -1077,13 +1198,13 @@ def get_system_state(filter=None): .. code-block:: bash salt '*' panos.get_system_state - salt '*' panos.get_system_state filter=cfg.ha.config.enabled - salt '*' panos.get_system_state filter=cfg.ha.* + salt '*' panos.get_system_state mask=cfg.ha.config.enabled + salt '*' panos.get_system_state mask=cfg.ha.* ''' - if filter: + if mask: query = {'type': 'op', - 'cmd': '{0}'.format(filter)} + 'cmd': '{0}'.format(mask)} else: query = {'type': 'op', 'cmd': ''} @@ -1093,6 +1214,7 @@ def get_system_state(filter=None): def get_uncommitted_changes(): ''' Retrieve a list of all uncommitted changes on the device. + Requires PANOS version 8.0.0 or greater. CLI Example: @@ -1101,6 +1223,10 @@ def get_uncommitted_changes(): salt '*' panos.get_uncommitted_changes ''' + _required_version = '8.0.0' + if not __proxy__['panos.is_required_version'](_required_version): + return False, 'The panos device requires version {0} or greater for this command.'.format(_required_version) + query = {'type': 'op', 'cmd': ''} @@ -1141,6 +1267,72 @@ def get_vlans(): return __proxy__['panos.call'](query) +def get_xpath(xpath=''): + ''' + Retrieve a specified xpath from the candidate configuration. + + xpath(str): The specified xpath in the candidate configuration. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_xpath /config/shared/service + + ''' + query = {'type': 'config', + 'action': 'get', + 'xpath': xpath} + + return __proxy__['panos.call'](query) + + +def get_zone(zone='', vsys='1'): + ''' + Get the candidate configuration for the specified zone. + + zone(str): The name of the zone. + + vsys(str): The string representation of the VSYS ID. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_zone trust + salt '*' panos.get_zone trust 2 + + ''' + query = {'type': 'config', + 'action': 'get', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/vsys/entry[@name=\'vsys{0}\']/' + 'zone/entry[@name=\'{1}\']'.format(vsys, zone)} + + return __proxy__['panos.call'](query) + + +def get_zones(vsys='1'): + ''' + Get all the zones in the candidate configuration. + + vsys(str): The string representation of the VSYS ID. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_zones + salt '*' panos.get_zones 2 + + ''' + query = {'type': 'config', + 'action': 'get', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/vsys/entry[@name=\'vsys{0}\']/' + 'zone'.format(vsys)} + + return __proxy__['panos.call'](query) + + def install_antivirus(version=None, latest=False, synch=False, skip_commit=False,): ''' Install anti-virus packages. diff --git a/salt/proxy/panos.py b/salt/proxy/panos.py index f7fb8f574c..86ef275a4f 100644 --- a/salt/proxy/panos.py +++ b/salt/proxy/panos.py @@ -191,7 +191,9 @@ from __future__ import absolute_import import logging # Import Salt Libs +from salt._compat import ElementTree as ET import salt.exceptions +import salt.utils.xmlutil as xml # This must be present or the Salt loader won't load this module. __proxyenabled__ = ['panos'] @@ -214,6 +216,22 @@ def __virtual__(): return __virtualname__ +def _strip_dirty(xmltree): + ''' + Removes dirtyID tags from the candidate config result. Palo Alto devices will make the candidate configuration with + a dirty ID after a change. This can cause unexpected results when parsing. + ''' + dirty = xmltree.attrib.pop('dirtyId', None) + if dirty: + xmltree.attrib.pop('admin', None) + xmltree.attrib.pop('time', None) + + for child in xmltree: + child = _strip_dirty(child) + + return xmltree + + def init(opts): ''' This function gets called when the proxy starts up. For @@ -271,7 +289,7 @@ def call(payload=None): ''' This function captures the query string and sends it to the Palo Alto device. ''' - ret = {} + r = None try: if DETAILS['method'] == 'dev_key': # Pass the api key without the target declaration @@ -280,11 +298,10 @@ def call(payload=None): r = __utils__['http.query'](DETAILS['url'], data=payload, method='POST', - decode_type='xml', + decode_type='plain', decode=True, verify_ssl=False, raise_error=True) - ret = r['dict'][0] elif DETAILS['method'] == 'dev_pass': # Pass credentials without the target declaration r = __utils__['http.query'](DETAILS['url'], @@ -292,11 +309,10 @@ def call(payload=None): password=DETAILS['password'], data=payload, method='POST', - decode_type='xml', + decode_type='plain', decode=True, verify_ssl=False, raise_error=True) - ret = r['dict'][0] elif DETAILS['method'] == 'pan_key': # Pass the api key with the target declaration conditional_payload = {'key': DETAILS['apikey'], @@ -305,11 +321,10 @@ def call(payload=None): r = __utils__['http.query'](DETAILS['url'], data=payload, method='POST', - decode_type='xml', + decode_type='plain', decode=True, verify_ssl=False, raise_error=True) - ret = r['dict'][0] elif DETAILS['method'] == 'pan_pass': # Pass credentials with the target declaration conditional_payload = {'target': DETAILS['serial']} @@ -319,14 +334,23 @@ def call(payload=None): password=DETAILS['password'], data=payload, method='POST', - decode_type='xml', + decode_type='plain', decode=True, verify_ssl=False, raise_error=True) - ret = r['dict'][0] except KeyError as err: raise salt.exceptions.CommandExecutionError("Did not receive a valid response from host.") - return ret + + if not r: + raise salt.exceptions.CommandExecutionError("Did not receive a valid response from host.") + + xmldata = ET.fromstring(r['text']) + + # If we are pulling the candidate configuration, we need to strip the dirtyId + if payload['type'] == 'config' and payload['action'] == 'get': + xmldata = (_strip_dirty(xmldata)) + + return xml.to_dict(xmldata, True) def is_required_version(required_version='0.0.0'): @@ -382,7 +406,7 @@ def grains(): DETAILS['grains_cache'] = GRAINS_CACHE try: query = {'type': 'op', 'cmd': ''} - DETAILS['grains_cache'] = call(query)['system'] + DETAILS['grains_cache'] = call(query)['result']['system'] except Exception as err: pass return DETAILS['grains_cache'] @@ -402,7 +426,7 @@ def ping(): ''' try: query = {'type': 'op', 'cmd': ''} - if 'system' in call(query): + if 'result' in call(query): return True else: return False diff --git a/salt/states/panos.py b/salt/states/panos.py index f941ff157a..c10cb5fbc9 100644 --- a/salt/states/panos.py +++ b/salt/states/panos.py @@ -87,6 +87,10 @@ greater than the passed version. For example, proxy['panos.is_required_version'] from __future__ import absolute_import import logging +# Import salt libs +import salt.utils.xmlutil as xml +from salt._compat import ElementTree as ET + log = logging.getLogger(__name__) @@ -135,7 +139,7 @@ def _edit_config(xpath, element): query = {'type': 'config', 'action': 'edit', 'xpath': xpath, - 'element': element} + 'element': element} response = __proxy__['panos.call'](query) @@ -239,21 +243,26 @@ def _validate_response(response): ''' if not response: - return False, "Error during move configuration. Verify connectivity to device." + return False, 'Unable to validate response from device.' elif 'msg' in response: - if response['msg'] == 'command succeeded': - return True, response['msg'] + if 'line' in response['msg']: + if response['msg']['line'] == 'already at the top': + return True, response + elif response['msg']['line'] == 'already at the bottom': + return True, response + else: + return False, response + elif response['msg'] == 'command succeeded': + return True, response else: - return False, response['msg'] - elif 'line' in response: - if response['line'] == 'already at the top': - return True, response['line'] - elif response['line'] == 'already at the bottom': - return True, response['line'] + return False, response + elif 'status' in response: + if response['status'] == "success": + return True, response else: - return False, response['line'] + return False, response else: - return False, "Error during move configuration. Verify connectivity to device." + return False, response def add_config_lock(name): @@ -280,6 +289,247 @@ def add_config_lock(name): return ret +def address_exists(name, + addressname=None, + vsys=1, + ipnetmask=None, + iprange=None, + fqdn=None, + description=None, + commit=False): + ''' + Ensures that an address object exists in the configured state. If it does not exist or is not configured with the + specified attributes, it will be adjusted to match the specified values. + + This module will only process a single address type (ip-netmask, ip-range, or fqdn). It will process the specified + value if the following order: ip-netmask, ip-range, fqdn. For proper execution, only specify a single address + type. + + name: The name of the module function to execute. + + addressname(str): The name of the address object. The name is case-sensitive and can have up to 31 characters, + which an be letters, numbers, spaces, hyphens, and underscores. The name must be unique on a firewall and, on + Panorama, unique within its device group and any ancestor or descendant device groups. + + vsys(str): The string representation of the VSYS ID. Defaults to VSYS 1. + + ipnetmask(str): The IPv4 or IPv6 address or IP address range using the format ip_address/mask or ip_address where + the mask is the number of significant binary digits used for the network portion of the address. Ideally, for IPv6, + you specify only the network portion, not the host portion. + + iprange(str): A range of addresses using the format ip_address–ip_address where both addresses can be IPv4 or both + can be IPv6. + + fqdn(str): A fully qualified domain name format. The FQDN initially resolves at commit time. Entries are + subsequently refreshed when the firewall performs a check every 30 minutes; all changes in the IP address for the + entries are picked up at the refresh cycle. + + description(str): A description for the policy (up to 255 characters). + + commit(bool): If true the firewall will commit the changes, if false do not commit changes. + + SLS Example: + + .. code-block:: yaml + + panos/address/h-10.10.10.10: + panos.address_exists: + - addressname: h-10.10.10.10 + - vsys: 1 + - ipnetmask: 10.10.10.10 + - commit: False + + panos/address/10.0.0.1-10.0.0.50: + panos.address_exists: + - addressname: r-10.0.0.1-10.0.0.50 + - vsys: 1 + - iprange: 10.0.0.1-10.0.0.50 + - commit: False + + panos/address/foo.bar.com: + panos.address_exists: + - addressname: foo.bar.com + - vsys: 1 + - fqdn: foo.bar.com + - description: My fqdn object + - commit: False + + ''' + ret = _default_ret(name) + + if not addressname: + ret.update({'comment': "The service name field must be provided."}) + return ret + + # Check if address object currently exists + address = __salt__['panos.get_address'](addressname, vsys)['result'] + + if address and 'entry' in address: + address = address['entry'] + else: + address = {} + + element = "" + + # Verify the arguments + if ipnetmask: + element = "{0}".format(ipnetmask) + elif iprange: + element = "{0}".format(iprange) + elif fqdn: + element = "{0}".format(fqdn) + else: + ret.update({'comment': "A valid address type must be specified."}) + return ret + + if description: + element += "{0}".format(description) + + full_element = "{1}".format(addressname, element) + + new_address = xml.to_dict(ET.fromstring(full_element), True) + + if address == new_address: + ret.update({ + 'comment': 'Address object already exists. No changes required.', + 'result': True + }) + return ret + else: + xpath = "/config/devices/entry[@name=\'localhost.localdomain\']/vsys/entry[@name=\'vsys{0}\']/address/" \ + "entry[@name=\'{1}\']".format(vsys, addressname) + + result, msg = _edit_config(xpath, full_element) + + if not result: + ret.update({ + 'comment': msg + }) + return ret + + if commit is True: + ret.update({ + 'changes': {'before': address, 'after': new_address}, + 'commit': __salt__['panos.commit'](), + 'comment': 'Address object successfully configured.', + 'result': True + }) + else: + ret.update({ + 'changes': {'before': address, 'after': new_address}, + 'comment': 'Service object successfully configured.', + 'result': True + }) + + return ret + + +def address_group_exists(name, + groupname=None, + vsys=1, + members=None, + description=None, + commit=False): + ''' + Ensures that an address group object exists in the configured state. If it does not exist or is not configured with + the specified attributes, it will be adjusted to match the specified values. + + This module will enforce group membership. If a group exists and contains members this state does not include, + those members will be removed and replaced with the specified members in the state. + + name: The name of the module function to execute. + + groupname(str): The name of the address group object. The name is case-sensitive and can have up to 31 characters, + which an be letters, numbers, spaces, hyphens, and underscores. The name must be unique on a firewall and, on + Panorama, unique within its device group and any ancestor or descendant device groups. + + vsys(str): The string representation of the VSYS ID. Defaults to VSYS 1. + + members(str, list): The members of the address group. These must be valid address objects or address groups on the + system that already exist prior to the execution of this state. + + description(str): A description for the policy (up to 255 characters). + + commit(bool): If true the firewall will commit the changes, if false do not commit changes. + + SLS Example: + + .. code-block:: yaml + + panos/address-group/my-group: + panos.address_group_exists: + - groupname: my-group + - vsys: 1 + - members: + - my-address-object + - my-other-address-group + - description: A group that needs to exist + - commit: False + + ''' + ret = _default_ret(name) + + if not groupname: + ret.update({'comment': "The group name field must be provided."}) + return ret + + # Check if address group object currently exists + group = __salt__['panos.get_address_group'](groupname, vsys)['result'] + + if group and 'entry' in group: + group = group['entry'] + else: + group = {} + + # Verify the arguments + if members: + element = "{0}".format(_build_members(members, True)) + else: + ret.update({'comment': "The group members must be provided."}) + return ret + + if description: + element += "{0}".format(description) + + full_element = "{1}".format(groupname, element) + + new_group = xml.to_dict(ET.fromstring(full_element), True) + + if group == new_group: + ret.update({ + 'comment': 'Address group object already exists. No changes required.', + 'result': True + }) + return ret + else: + xpath = "/config/devices/entry[@name=\'localhost.localdomain\']/vsys/entry[@name=\'vsys{0}\']/address-group/" \ + "entry[@name=\'{1}\']".format(vsys, groupname) + + result, msg = _edit_config(xpath, full_element) + + if not result: + ret.update({ + 'comment': msg + }) + return ret + + if commit is True: + ret.update({ + 'changes': {'before': group, 'after': new_group}, + 'commit': __salt__['panos.commit'](), + 'comment': 'Address group object successfully configured.', + 'result': True + }) + else: + ret.update({ + 'changes': {'before': group, 'after': new_group}, + 'comment': 'Address group object successfully configured.', + 'result': True + }) + + return ret + + def clone_config(name, xpath=None, newname=None, commit=False): ''' Clone a specific XPATH and set it to a new name. @@ -317,13 +567,16 @@ def clone_config(name, xpath=None, newname=None, commit=False): 'xpath': xpath, 'newname': newname} - response = __proxy__['panos.call'](query) + result, response = _validate_response(__proxy__['panos.call'](query)) ret.update({ 'changes': response, - 'result': True + 'result': result }) + if not result: + return ret + if commit is True: ret.update({ 'commit': __salt__['panos.commit'](), @@ -386,15 +639,18 @@ def delete_config(name, xpath=None, commit=False): query = {'type': 'config', 'action': 'delete', - 'xpath': xpath} + 'xpath': xpath} - response = __proxy__['panos.call'](query) + result, response = _validate_response(__proxy__['panos.call'](query)) ret.update({ 'changes': response, - 'result': True + 'result': result }) + if not result: + return ret + if commit is True: ret.update({ 'commit': __salt__['panos.commit'](), @@ -434,7 +690,7 @@ def download_software(name, version=None, synch=False, check=False): if check is True: __salt__['panos.check_software']() - versions = __salt__['panos.get_software_info']() + versions = __salt__['panos.get_software_info']()['result'] if 'sw-updates' not in versions \ or 'versions' not in versions['sw-updates'] \ @@ -457,7 +713,7 @@ def download_software(name, version=None, synch=False, check=False): 'changes': __salt__['panos.download_software_version'](version=version, synch=synch) }) - versions = __salt__['panos.get_software_info']() + versions = __salt__['panos.get_software_info']()['result'] if 'sw-updates' not in versions \ or 'versions' not in versions['sw-updates'] \ @@ -508,6 +764,32 @@ def edit_config(name, xpath=None, value=None, commit=False): ''' ret = _default_ret(name) + # Verify if the current XPATH is equal to the specified value. + # If we are equal, no changes required. + xpath_split = xpath.split("/") + + # Retrieve the head of the xpath for validation. + if len(xpath_split) > 0: + head = xpath_split[-1] + if "[" in head: + head = head.split("[")[0] + + current_element = __salt__['panos.get_xpath'](xpath)['result'] + + if head and current_element and head in current_element: + current_element = current_element[head] + else: + current_element = {} + + new_element = xml.to_dict(ET.fromstring(value), True) + + if current_element == new_element: + ret.update({ + 'comment': 'XPATH is already equal to the specified value.', + 'result': True + }) + return ret + result, msg = _edit_config(xpath, value) ret.update({ @@ -515,15 +797,20 @@ def edit_config(name, xpath=None, value=None, commit=False): 'result': result }) - # Ensure we do not commit after a failed action if not result: return ret if commit is True: ret.update({ + 'changes': {'before': current_element, 'after': new_element}, 'commit': __salt__['panos.commit'](), 'result': True }) + else: + ret.update({ + 'changes': {'before': current_element, 'after': new_element}, + 'result': True + }) return ret @@ -585,7 +872,8 @@ def move_config(name, xpath=None, where=None, dst=None, commit=False): result, msg = _move_bottom(xpath) ret.update({ - 'result': result + 'result': result, + 'comment': msg }) if not result: @@ -660,13 +948,16 @@ def rename_config(name, xpath=None, newname=None, commit=False): 'xpath': xpath, 'newname': newname} - response = __proxy__['panos.call'](query) + result, response = _validate_response(__proxy__['panos.call'](query)) ret.update({ 'changes': response, - 'result': True + 'result': result }) + if not result: + return ret + if commit is True: ret.update({ 'commit': __salt__['panos.commit'](), @@ -854,7 +1145,12 @@ def security_rule_exists(name, return ret # Check if rule currently exists - rule = __salt__['panos.get_security_rule'](rulename, vsys) + rule = __salt__['panos.get_security_rule'](rulename, vsys)['result'] + + if rule and 'entry' in rule: + rule = rule['entry'] + else: + rule = {} # Build the rule element element = "" @@ -964,29 +1260,32 @@ def security_rule_exists(name, full_element = "{1}".format(rulename, element) - create_rule = False + new_rule = xml.to_dict(ET.fromstring(full_element), True) - if 'result' in rule: - if rule['result'] == "None": - create_rule = True + config_change = False - if create_rule: - xpath = "/config/devices/entry[@name=\'localhost.localdomain\']/vsys/entry[@name=\'vsys{0}\']/rulebase/" \ - "security/rules".format(vsys) - - result, msg = _set_config(xpath, full_element) - if not result: - ret['changes']['set'] = msg - return ret + if rule == new_rule: + ret.update({ + 'comment': 'Security rule already exists. No changes required.' + }) else: + config_change = True xpath = "/config/devices/entry[@name=\'localhost.localdomain\']/vsys/entry[@name=\'vsys{0}\']/rulebase/" \ "security/rules/entry[@name=\'{1}\']".format(vsys, rulename) result, msg = _edit_config(xpath, full_element) + if not result: - ret['changes']['edit'] = msg + ret.update({ + 'comment': msg + }) return ret + ret.update({ + 'changes': {'before': rule, 'after': new_rule}, + 'comment': 'Security rule verified successfully.' + }) + if move: movepath = "/config/devices/entry[@name=\'localhost.localdomain\']/vsys/entry[@name=\'vsys{0}\']/rulebase/" \ "security/rules/entry[@name=\'{1}\']".format(vsys, rulename) @@ -1001,19 +1300,244 @@ def security_rule_exists(name, elif move == "bottom": move_result, move_msg = _move_bottom(movepath) + if config_change: + ret.update({ + 'changes': {'before': rule, 'after': new_rule, 'move': move_msg} + }) + else: + ret.update({ + 'changes': {'move': move_msg} + }) + if not move_result: - ret['changes']['move'] = move_msg + ret.update({ + 'comment': move_msg + }) return ret if commit is True: ret.update({ 'commit': __salt__['panos.commit'](), - 'comment': 'Security rule verified successfully.', 'result': True }) else: ret.update({ - 'comment': 'Security rule verified successfully.', + 'result': True + }) + + return ret + + +def service_exists(name, servicename=None, vsys=1, protocol=None, port=None, description=None, commit=False): + ''' + Ensures that a service object exists in the configured state. If it does not exist or is not configured with the + specified attributes, it will be adjusted to match the specified values. + + name: The name of the module function to execute. + + servicename(str): The name of the security object. The name is case-sensitive and can have up to 31 characters, + which an be letters, numbers, spaces, hyphens, and underscores. The name must be unique on a firewall and, on + Panorama, unique within its device group and any ancestor or descendant device groups. + + vsys(str): The string representation of the VSYS ID. Defaults to VSYS 1. + + protocol(str): The protocol that is used by the service object. The only valid options are tcp and udp. + + port(str): The port number that is used by the service object. This can be specified as a single integer or a + valid range of ports. + + description(str): A description for the policy (up to 255 characters). + + commit(bool): If true the firewall will commit the changes, if false do not commit changes. + + SLS Example: + + .. code-block:: yaml + + panos/service/tcp-80: + panos.service_exists: + - servicename: tcp-80 + - vsys: 1 + - protocol: tcp + - port: 80 + - description: Hypertext Transfer Protocol + - commit: False + + panos/service/udp-500-550: + panos.service_exists: + - servicename: udp-500-550 + - vsys: 3 + - protocol: udp + - port: 500-550 + - commit: False + + ''' + ret = _default_ret(name) + + if not servicename: + ret.update({'comment': "The service name field must be provided."}) + return ret + + # Check if service object currently exists + service = __salt__['panos.get_service'](servicename, vsys)['result'] + + if service and 'entry' in service: + service = service['entry'] + else: + service = {} + + # Verify the arguments + if not protocol and protocol not in ['tcp', 'udp']: + ret.update({'comment': "The protocol must be provided and must be tcp or udp."}) + return ret + if not port: + ret.update({'comment': "The port field must be provided."}) + return ret + + element = "<{0}>{1}".format(protocol, port) + + if description: + element += "{0}".format(description) + + full_element = "{1}".format(servicename, element) + + new_service = xml.to_dict(ET.fromstring(full_element), True) + + if service == new_service: + ret.update({ + 'comment': 'Service object already exists. No changes required.', + 'result': True + }) + return ret + else: + xpath = "/config/devices/entry[@name=\'localhost.localdomain\']/vsys/entry[@name=\'vsys{0}\']/service/" \ + "entry[@name=\'{1}\']".format(vsys, servicename) + + result, msg = _edit_config(xpath, full_element) + + if not result: + ret.update({ + 'comment': msg + }) + return ret + + if commit is True: + ret.update({ + 'changes': {'before': service, 'after': new_service}, + 'commit': __salt__['panos.commit'](), + 'comment': 'Service object successfully configured.', + 'result': True + }) + else: + ret.update({ + 'changes': {'before': service, 'after': new_service}, + 'comment': 'Service object successfully configured.', + 'result': True + }) + + return ret + + +def service_group_exists(name, + groupname=None, + vsys=1, + members=None, + description=None, + commit=False): + ''' + Ensures that a service group object exists in the configured state. If it does not exist or is not configured with + the specified attributes, it will be adjusted to match the specified values. + + This module will enforce group membership. If a group exists and contains members this state does not include, + those members will be removed and replaced with the specified members in the state. + + name: The name of the module function to execute. + + groupname(str): The name of the service group object. The name is case-sensitive and can have up to 31 characters, + which an be letters, numbers, spaces, hyphens, and underscores. The name must be unique on a firewall and, on + Panorama, unique within its device group and any ancestor or descendant device groups. + + vsys(str): The string representation of the VSYS ID. Defaults to VSYS 1. + + members(str, list): The members of the service group. These must be valid service objects or service groups on the + system that already exist prior to the execution of this state. + + description(str): A description for the policy (up to 255 characters). + + commit(bool): If true the firewall will commit the changes, if false do not commit changes. + + SLS Example: + + .. code-block:: yaml + + panos/service-group/my-group: + panos.service_group_exists: + - groupname: my-group + - vsys: 1 + - members: + - tcp-80 + - custom-port-group + - description: A group that needs to exist + - commit: False + + ''' + ret = _default_ret(name) + + if not groupname: + ret.update({'comment': "The group name field must be provided."}) + return ret + + # Check if service group object currently exists + group = __salt__['panos.get_service_group'](groupname, vsys)['result'] + + if group and 'entry' in group: + group = group['entry'] + else: + group = {} + + # Verify the arguments + if members: + element = "{0}".format(_build_members(members, True)) + else: + ret.update({'comment': "The group members must be provided."}) + return ret + + if description: + element += "{0}".format(description) + + full_element = "{1}".format(groupname, element) + + new_group = xml.to_dict(ET.fromstring(full_element), True) + + if group == new_group: + ret.update({ + 'comment': 'Service group object already exists. No changes required.', + 'result': True + }) + return ret + else: + xpath = "/config/devices/entry[@name=\'localhost.localdomain\']/vsys/entry[@name=\'vsys{0}\']/service-group/" \ + "entry[@name=\'{1}\']".format(vsys, groupname) + + result, msg = _edit_config(xpath, full_element) + + if not result: + ret.update({ + 'comment': msg + }) + return ret + + if commit is True: + ret.update({ + 'changes': {'before': group, 'after': new_group}, + 'commit': __salt__['panos.commit'](), + 'comment': 'Service group object successfully configured.', + 'result': True + }) + else: + ret.update({ + 'changes': {'before': group, 'after': new_group}, + 'comment': 'Service group object successfully configured.', 'result': True }) @@ -1056,7 +1580,6 @@ def set_config(name, xpath=None, value=None, commit=False): 'result': result }) - # Ensure we do not commit after a failed action if not result: return ret diff --git a/salt/utils/xmlutil.py b/salt/utils/xmlutil.py index bb07921aaa..2f32985e2e 100644 --- a/salt/utils/xmlutil.py +++ b/salt/utils/xmlutil.py @@ -7,34 +7,37 @@ Various XML utilities from __future__ import absolute_import -def to_dict(xmltree): +def _conv_name(x): ''' - Convert an XML tree into a dict. The tree that is passed in must be an - ElementTree object. + If this XML tree has an xmlns attribute, then etree will add it + to the beginning of the tag, like: "{http://path}tag". + ''' + if '}' in x: + comps = x.split('}') + name = comps[1] + return name + return x + + +def _to_dict(xmltree): + ''' + Converts an XML ElementTree to a dictionary that only contains items. + This is the default behavior in version 2017.7. This will default to prevent + unexpected parsing issues on modules dependant on this. ''' # If this object has no children, the for..loop below will return nothing # for it, so just return a single dict representing it. - if len(xmltree.getchildren()) < 1 and len(xmltree.attrib.items()) < 1: - name = xmltree.tag - if '}' in name: - comps = name.split('}') - name = comps[1] + if len(xmltree.getchildren()) < 1: + name = _conv_name(xmltree.tag) return {name: xmltree.text} xmldict = {} for item in xmltree: - name = item.tag - if '}' in name: - # If this XML tree has an xmlns attribute, then etree will add it - # to the beginning of the tag, like: "{http://path}tag". This - # aggression will not stand, man. - comps = name.split('}') - name = comps[1] + name = _conv_name(item.tag) + if name not in xmldict: if len(item.getchildren()) > 0: - xmldict[name] = to_dict(item) - elif len(item.attrib.items()) > 0: - xmldict[name] = to_dict(item) + xmldict[name] = _to_dict(item) else: xmldict[name] = item.text else: @@ -43,13 +46,56 @@ def to_dict(xmltree): # to happen, and behave accordingly. if not isinstance(xmldict[name], list): xmldict[name] = [xmldict[name]] - xmldict[name].append(to_dict(item)) + xmldict[name].append(_to_dict(item)) + return xmldict + + +def _to_full_dict(xmltree): + ''' + Returns the full XML dictionary including attributes. + ''' + xmldict = {} for attrName, attrValue in xmltree.attrib.items(): - if attrName not in xmldict: - xmldict[attrName] = attrValue + xmldict[attrName] = attrValue + + if len(xmltree.getchildren()) < 1: + if len(xmldict) == 0: + # If we don't have attributes, we should return the value as a string + # ex: test + return xmltree.text + elif xmltree.text: + # XML allows for empty sets with attributes, so we need to make sure that capture this. + # ex: + xmldict[_conv_name(xmltree.tag)] = xmltree.text + + for item in xmltree: + name = _conv_name(item.tag) + + if name not in xmldict: + xmldict[name] = _to_full_dict(item) else: - # Attempt to ensure that items are not overwritten by attributes. - xmldict["attr{0}".format(attrName)] = attrValue + # If a tag appears more than once in the same place, convert it to + # a list. This may require that the caller watch for such a thing + # to happen, and behave accordingly. + if not isinstance(xmldict[name], list): + xmldict[name] = [xmldict[name]] + + xmldict[name].append(_to_full_dict(item)) return xmldict + + +def to_dict(xmltree, attr=False): + ''' + Convert an XML tree into a dict. The tree that is passed in must be an + ElementTree object. + Args: + xmltree: An ElementTree object. + attr: If true, attributes will be parsed. If false, they will be ignored. + + ''' + if attr: + return _to_full_dict(xmltree) + else: + return _to_dict(xmltree) From f7df41fa9411da7cc96808680e8a65e9b99e753e Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 26 Sep 2017 12:59:23 -0600 Subject: [PATCH 006/184] split build and install for pkg osx --- pkg/osx/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/osx/build.sh b/pkg/osx/build.sh index 7850d48cd8..fcf4b4e061 100755 --- a/pkg/osx/build.sh +++ b/pkg/osx/build.sh @@ -88,7 +88,8 @@ sudo $PKGRESOURCES/build_env.sh $PYVER echo -n -e "\033]0;Build: Install Salt\007" sudo rm -rf $SRCDIR/build sudo rm -rf $SRCDIR/dist -sudo $PYTHON $SRCDIR/setup.py build -e "$PYTHON -E -s" install +sudo $PYTHON $SRCDIR/setup.py build -e "$PYTHON -E -s" +sudo $PYTHON $SRCDIR/setup.py install ############################################################################ # Build Package From 12845ae80239c49ba9ae842d9ca64f41d1394688 Mon Sep 17 00:00:00 2001 From: Andrei Belov Date: Wed, 20 Sep 2017 08:07:13 +0300 Subject: [PATCH 007/184] Several fixes for RDS DB parameter group management In particular: - it is now possible to manage all the parameters in a group, without limiting to MaxRecords=100 (thanks to pagination); - update_parameter_group() now composes valid JSON payload, automatically substitutes boolean values to 'on' / 'off' strings; - parameter_present() now shows actual error message produced by ModifyDBParameterGroup API call. --- salt/modules/boto_rds.py | 35 ++++++++++++++++++++--------------- salt/states/boto_rds.py | 13 ++++++++----- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/salt/modules/boto_rds.py b/salt/modules/boto_rds.py index 71e694cfa3..fcaed4f8b8 100644 --- a/salt/modules/boto_rds.py +++ b/salt/modules/boto_rds.py @@ -493,10 +493,17 @@ def update_parameter_group(name, parameters, apply_method="pending-reboot", param_list = [] for key, value in six.iteritems(parameters): - item = (key, value, apply_method) + item = odict.OrderedDict() + item.update({'ParameterName': key}) + item.update({'ApplyMethod': apply_method}) + if type(value) is bool: + item.update({'ParameterValue': 'on' if value else 'off'}) + else: + item.update({'ParameterValue': str(value)}) param_list.append(item) - if not len(param_list): - return {'results': False} + + if not len(param_list): + return {'results': False} try: conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) @@ -781,6 +788,7 @@ def describe_parameters(name, Source=None, MaxRecords=None, Marker=None, 'message': 'Could not establish a connection to RDS'} kwargs = {} + kwargs.update({'DBParameterGroupName': name}) for key in ('Marker', 'Source'): if locals()[key] is not None: kwargs[key] = str(locals()[key]) @@ -788,26 +796,23 @@ def describe_parameters(name, Source=None, MaxRecords=None, Marker=None, if locals()['MaxRecords'] is not None: kwargs['MaxRecords'] = int(locals()['MaxRecords']) - r = conn.describe_db_parameters(DBParameterGroupName=name, **kwargs) + pag = conn.get_paginator('describe_db_parameters') + pit = pag.paginate(**kwargs) - if not r: - return {'result': False, - 'message': 'Failed to get RDS parameters for group {0}.' - .format(name)} - - results = r['Parameters'] keys = ['ParameterName', 'ParameterValue', 'Description', 'Source', 'ApplyType', 'DataType', 'AllowedValues', 'IsModifieable', 'MinimumEngineVersion', 'ApplyMethod'] parameters = odict.OrderedDict() ret = {'result': True} - for result in results: - data = odict.OrderedDict() - for k in keys: - data[k] = result.get(k) - parameters[result.get('ParameterName')] = data + for p in pit: + for result in p['Parameters']: + data = odict.OrderedDict() + for k in keys: + data[k] = result.get(k) + + parameters[result.get('ParameterName')] = data ret['parameters'] = parameters return ret diff --git a/salt/states/boto_rds.py b/salt/states/boto_rds.py index 15163add8a..a2f6065187 100644 --- a/salt/states/boto_rds.py +++ b/salt/states/boto_rds.py @@ -692,7 +692,10 @@ def parameter_present(name, db_parameter_group_family, description, parameters=N changed = {} for items in parameters: for k, value in items.items(): - params[k] = value + if type(value) is bool: + params[k] = 'on' if value else 'off' + else: + params[k] = str(value) logging.debug('Parameters from user are : {0}.'.format(params)) options = __salt__['boto_rds.describe_parameters'](name=name, region=region, key=key, keyid=keyid, profile=profile) if not options.get('result'): @@ -700,8 +703,8 @@ def parameter_present(name, db_parameter_group_family, description, parameters=N ret['comment'] = os.linesep.join([ret['comment'], 'Faled to get parameters for group {0}.'.format(name)]) return ret for parameter in options['parameters'].values(): - if parameter['ParameterName'] in params and str(params.get(parameter['ParameterName'])) != str(parameter['ParameterValue']): - logging.debug('Values that are being compared are {0}:{1} .'.format(params.get(parameter['ParameterName']), parameter['ParameterValue'])) + if parameter['ParameterName'] in params and params.get(parameter['ParameterName']) != str(parameter['ParameterValue']): + logging.debug('Values that are being compared for {0} are {1}:{2} .'.format(parameter['ParameterName'], params.get(parameter['ParameterName']), parameter['ParameterValue'])) changed[parameter['ParameterName']] = params.get(parameter['ParameterName']) if len(changed) > 0: if __opts__['test']: @@ -710,9 +713,9 @@ def parameter_present(name, db_parameter_group_family, description, parameters=N return ret update = __salt__['boto_rds.update_parameter_group'](name, parameters=changed, apply_method=apply_method, tags=tags, region=region, key=key, keyid=keyid, profile=profile) - if not update: + if 'error' in update: ret['result'] = False - ret['comment'] = os.linesep.join([ret['comment'], 'Failed to change parameters {0} for group {1}.'.format(changed, name)]) + ret['comment'] = os.linesep.join([ret['comment'], 'Failed to change parameters {0} for group {1}:'.format(changed, name), update['error']['message']]) return ret ret['changes']['Parameters'] = changed ret['comment'] = os.linesep.join([ret['comment'], 'Parameters {0} for group {1} are changed.'.format(changed, name)]) From 0f50f0c926966ffe1b6d05e58a272f91097e4b91 Mon Sep 17 00:00:00 2001 From: Tom Williams Date: Fri, 29 Sep 2017 14:13:02 -0400 Subject: [PATCH 008/184] INFRA-5669 - add retry loops for API rate limiting in ELB health_check functions --- salt/modules/boto_elb.py | 60 +++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/salt/modules/boto_elb.py b/salt/modules/boto_elb.py index eed2759ca0..7c427cb889 100644 --- a/salt/modules/boto_elb.py +++ b/salt/modules/boto_elb.py @@ -644,22 +644,29 @@ def get_health_check(name, region=None, key=None, keyid=None, profile=None): salt myminion boto_elb.get_health_check myelb ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + retries = 30 - try: - lb = conn.get_all_load_balancers(load_balancer_names=[name]) - lb = lb[0] - ret = odict.OrderedDict() - hc = lb.health_check - ret['interval'] = hc.interval - ret['target'] = hc.target - ret['healthy_threshold'] = hc.healthy_threshold - ret['timeout'] = hc.timeout - ret['unhealthy_threshold'] = hc.unhealthy_threshold - return ret - except boto.exception.BotoServerError as error: - log.debug(error) - log.error('ELB {0} does not exist: {1}'.format(name, error)) - return {} + while True: + try: + lb = conn.get_all_load_balancers(load_balancer_names=[name]) + lb = lb[0] + ret = odict.OrderedDict() + hc = lb.health_check + ret['interval'] = hc.interval + ret['target'] = hc.target + ret['healthy_threshold'] = hc.healthy_threshold + ret['timeout'] = hc.timeout + ret['unhealthy_threshold'] = hc.unhealthy_threshold + return ret + except boto.exception.BotoServerError as e: + if retries and e.code == 'Throttling': + log.debug('Throttled by AWS API, will retry in 5 seconds.') + time.sleep(5) + retries -= 1 + continue + log.error(error) + log.error('ELB {0} not found.'.format(name)) + return {} def set_health_check(name, health_check, region=None, key=None, keyid=None, @@ -674,16 +681,23 @@ def set_health_check(name, health_check, region=None, key=None, keyid=None, salt myminion boto_elb.set_health_check myelb '{"target": "HTTP:80/"}' ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + retries = 30 hc = HealthCheck(**health_check) - try: - conn.configure_health_check(name, hc) - log.info('Configured health check on ELB {0}'.format(name)) - except boto.exception.BotoServerError as error: - log.debug(error) - log.info('Failed to configure health check on ELB {0}: {1}'.format(name, error)) - return False - return True + while True: + try: + conn.configure_health_check(name, hc) + log.info('Configured health check on ELB {0}'.format(name)) + return True + except boto.exception.BotoServerError as error: + if retries and e.code == 'Throttling': + log.debug('Throttled by AWS API, will retry in 5 seconds.') + time.sleep(5) + retries -= 1 + continue + log.error(error) + log.error('Failed to configure health check on ELB {0}'.format(name)) + return False def register_instances(name, instances, region=None, key=None, keyid=None, From 0f3503ee22f50fe8caa31b87a8fa7e51e410a534 Mon Sep 17 00:00:00 2001 From: spenceation Date: Fri, 29 Sep 2017 14:06:38 -0400 Subject: [PATCH 009/184] Added unit test for utils/xmlutil. --- tests/unit/utils/test_xmlutil.py | 147 +++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 tests/unit/utils/test_xmlutil.py diff --git a/tests/unit/utils/test_xmlutil.py b/tests/unit/utils/test_xmlutil.py new file mode 100644 index 0000000000..2377ad782c --- /dev/null +++ b/tests/unit/utils/test_xmlutil.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- +''' + tests.unit.xmlutil_test + ~~~~~~~~~~~~~~~~~~~~ +''' +# Import Salt Testing libs +from tests.support.unit import TestCase + +# Import Salt libs +from salt._compat import ElementTree as ET +import salt.utils.xmlutil as xml + + +class XMLUtilTestCase(TestCase): + ''' + Tests that salt.utils.xmlutil properly parses XML data and returns as a properly formatted + dictionary. The default method of parsing will ignore attributes and return only the child + items. The full method will include parsing attributes. + ''' + + def setUp(self): + + # Populate our use cases for specific XML formats. + self.cases = { + 'a': { + 'xml': 'data', + 'legacy': {'parent': 'data'}, + 'full': 'data' + }, + 'b': { + 'xml': 'data', + 'legacy': {'parent': 'data'}, + 'full': {'parent': 'data', 'value': 'data'} + }, + 'c': { + 'xml': 'datadata' + '', + 'legacy': {'child': ['data', {'child': 'data'}, {'child': None}, {'child': None}]}, + 'full': {'child': ['data', {'child': 'data', 'value': 'data'}, {'value': 'data'}, None]} + }, + 'd': { + 'xml': 'data', + 'legacy': {'child': 'data'}, + 'full': {'child': 'data', 'another': 'data', 'value': 'data'} + }, + 'e': { + 'xml': 'data', + 'legacy': {'child': 'data'}, + 'full': {'child': {'child': 'data', 'value': 'data'}, 'another': 'data', 'value': 'data'} + }, + 'f': { + 'xml': 'data' + 'data', + 'legacy': {'child': [{'sub-child': 'data'}, {'child': 'data'}]}, + 'full': {'child': [{'sub-child': {'value': 'data', 'sub-child': 'data'}}, 'data']} + }, + } + + def test_xml_case_a(self): + xmldata = ET.fromstring(self.cases['a']['xml']) + defaultdict = xml.to_dict(xmldata) + self.assertEqual(defaultdict, self.cases['a']['legacy']) + + def test_xml_case_a_legacy(self): + xmldata = ET.fromstring(self.cases['a']['xml']) + defaultdict = xml.to_dict(xmldata, False) + self.assertEqual(defaultdict, self.cases['a']['legacy']) + + def test_xml_case_a_full(self): + xmldata = ET.fromstring(self.cases['a']['xml']) + defaultdict = xml.to_dict(xmldata, True) + self.assertEqual(defaultdict, self.cases['a']['full']) + + def test_xml_case_b(self): + xmldata = ET.fromstring(self.cases['b']['xml']) + defaultdict = xml.to_dict(xmldata) + self.assertEqual(defaultdict, self.cases['b']['legacy']) + + def test_xml_case_b_legacy(self): + xmldata = ET.fromstring(self.cases['b']['xml']) + defaultdict = xml.to_dict(xmldata, False) + self.assertEqual(defaultdict, self.cases['b']['legacy']) + + def test_xml_case_b_full(self): + xmldata = ET.fromstring(self.cases['b']['xml']) + defaultdict = xml.to_dict(xmldata, True) + self.assertEqual(defaultdict, self.cases['b']['full']) + + def test_xml_case_c(self): + xmldata = ET.fromstring(self.cases['c']['xml']) + defaultdict = xml.to_dict(xmldata) + self.assertEqual(defaultdict, self.cases['c']['legacy']) + + def test_xml_case_c_legacy(self): + xmldata = ET.fromstring(self.cases['c']['xml']) + defaultdict = xml.to_dict(xmldata, False) + self.assertEqual(defaultdict, self.cases['c']['legacy']) + + def test_xml_case_c_full(self): + xmldata = ET.fromstring(self.cases['c']['xml']) + defaultdict = xml.to_dict(xmldata, True) + self.assertEqual(defaultdict, self.cases['c']['full']) + + def test_xml_case_d(self): + xmldata = ET.fromstring(self.cases['d']['xml']) + defaultdict = xml.to_dict(xmldata) + self.assertEqual(defaultdict, self.cases['d']['legacy']) + + def test_xml_case_d_legacy(self): + xmldata = ET.fromstring(self.cases['d']['xml']) + defaultdict = xml.to_dict(xmldata, False) + self.assertEqual(defaultdict, self.cases['d']['legacy']) + + def test_xml_case_d_full(self): + xmldata = ET.fromstring(self.cases['d']['xml']) + defaultdict = xml.to_dict(xmldata, True) + self.assertEqual(defaultdict, self.cases['d']['full']) + + def test_xml_case_e(self): + xmldata = ET.fromstring(self.cases['e']['xml']) + defaultdict = xml.to_dict(xmldata) + self.assertEqual(defaultdict, self.cases['e']['legacy']) + + def test_xml_case_e_legacy(self): + xmldata = ET.fromstring(self.cases['e']['xml']) + defaultdict = xml.to_dict(xmldata, False) + self.assertEqual(defaultdict, self.cases['e']['legacy']) + + def test_xml_case_e_full(self): + xmldata = ET.fromstring(self.cases['e']['xml']) + defaultdict = xml.to_dict(xmldata, True) + self.assertEqual(defaultdict, self.cases['e']['full']) + + def test_xml_case_f(self): + xmldata = ET.fromstring(self.cases['f']['xml']) + defaultdict = xml.to_dict(xmldata) + self.assertEqual(defaultdict, self.cases['f']['legacy']) + + def test_xml_case_f_legacy(self): + xmldata = ET.fromstring(self.cases['f']['xml']) + defaultdict = xml.to_dict(xmldata, False) + self.assertEqual(defaultdict, self.cases['f']['legacy']) + + def test_xml_case_f_full(self): + xmldata = ET.fromstring(self.cases['f']['xml']) + defaultdict = xml.to_dict(xmldata, True) + self.assertEqual(defaultdict, self.cases['f']['full']) From 993ea72f753f994e27982d90fde211fe91936d5f Mon Sep 17 00:00:00 2001 From: Nasenbaer Date: Fri, 16 Sep 2016 13:07:06 +0200 Subject: [PATCH 010/184] Use dsn if specified as all recent raven libraries can init via dsn --- salt/log/handlers/sentry_mod.py | 37 ++++++++++++--------------------- 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/salt/log/handlers/sentry_mod.py b/salt/log/handlers/sentry_mod.py index 496c9929a1..81625b158d 100644 --- a/salt/log/handlers/sentry_mod.py +++ b/salt/log/handlers/sentry_mod.py @@ -123,36 +123,25 @@ def setup_handlers(): url = urlparse(dsn) if not transport_registry.supported_scheme(url.scheme): raise ValueError('Unsupported Sentry DSN scheme: {0}'.format(url.scheme)) - dsn_config = {} - if (hasattr(transport_registry, 'compute_scope') and - callable(transport_registry.compute_scope)): - conf_extras = transport_registry.compute_scope(url, dsn_config) - dsn_config.update(conf_extras) - options.update({ - 'project': dsn_config['SENTRY_PROJECT'], - 'servers': dsn_config['SENTRY_SERVERS'], - 'public_key': dsn_config['SENTRY_PUBLIC_KEY'], - 'secret_key': dsn_config['SENTRY_SECRET_KEY'] - }) except ValueError as exc: log.info( 'Raven failed to parse the configuration provided ' 'DSN: {0}'.format(exc) ) - # Allow options to be overridden if previously parsed, or define them - for key in ('project', 'servers', 'public_key', 'secret_key'): - config_value = get_config_value(key) - if config_value is None and key not in options: - log.debug( - 'The required \'sentry_handler\' configuration key, ' - '\'{0}\', is not properly configured. Not configuring ' - 'the sentry logging handler.'.format(key) - ) - return - elif config_value is None: - continue - options[key] = config_value + if not dsn: + for key in ('project', 'servers', 'public_key', 'secret_key'): + config_value = get_config_value(key) + if config_value is None and key not in options: + log.debug( + 'The required \'sentry_handler\' configuration key, ' + '\'{0}\', is not properly configured. Not configuring ' + 'the sentry logging handler.'.format(key) + ) + return + elif config_value is None: + continue + options[key] = config_value # site: An optional, arbitrary string to identify this client installation. options.update({ From 5df8e597961eba4887a668b2b18df55b734a603f Mon Sep 17 00:00:00 2001 From: Bener Date: Fri, 29 Sep 2017 13:58:11 +0300 Subject: [PATCH 011/184] Add OpsGenie's execution and state modules --- doc/ref/modules/all/index.rst | 1 + doc/ref/modules/all/salt.modules.opsgenie.rst | 6 + doc/ref/states/all/index.rst | 1 + doc/ref/states/all/salt.states.opsgenie.rst | 6 + salt/modules/opsgenie.py | 92 +++++++++++ salt/states/opsgenie.py | 153 ++++++++++++++++++ 6 files changed, 259 insertions(+) create mode 100644 doc/ref/modules/all/salt.modules.opsgenie.rst create mode 100644 doc/ref/states/all/salt.states.opsgenie.rst create mode 100644 salt/modules/opsgenie.py create mode 100644 salt/states/opsgenie.py diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index b9f4b3f35c..a8be479a31 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -299,6 +299,7 @@ execution modules openstack_mng openvswitch opkg + opsgenie oracle osquery out diff --git a/doc/ref/modules/all/salt.modules.opsgenie.rst b/doc/ref/modules/all/salt.modules.opsgenie.rst new file mode 100644 index 0000000000..e16999297e --- /dev/null +++ b/doc/ref/modules/all/salt.modules.opsgenie.rst @@ -0,0 +1,6 @@ +=================== +salt.modules.opsgenie +=================== + +.. automodule:: salt.modules.opsgenie + :members: diff --git a/doc/ref/states/all/index.rst b/doc/ref/states/all/index.rst index 0b681ace7e..3a140b2e5c 100644 --- a/doc/ref/states/all/index.rst +++ b/doc/ref/states/all/index.rst @@ -188,6 +188,7 @@ state modules openstack_config openvswitch_bridge openvswitch_port + opsgenie pagerduty pagerduty_escalation_policy pagerduty_schedule diff --git a/doc/ref/states/all/salt.states.opsgenie.rst b/doc/ref/states/all/salt.states.opsgenie.rst new file mode 100644 index 0000000000..f5b0561f3e --- /dev/null +++ b/doc/ref/states/all/salt.states.opsgenie.rst @@ -0,0 +1,6 @@ +===================== +salt.states.opsgenie +===================== + +.. automodule:: salt.states.opsgenie + :members: diff --git a/salt/modules/opsgenie.py b/salt/modules/opsgenie.py new file mode 100644 index 0000000000..cfde15b90e --- /dev/null +++ b/salt/modules/opsgenie.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +''' +Module for sending data to OpsGenie + +.. versionadded:: Oxygen + +:configuration: This module can be used in Reactor System for + posting data to OpsGenie as a remote-execution function. + For example: + .. code-block:: yaml + opsgenie_event_poster: + local.opsgenie.post_data: + - tgt: 'salt-minion' + - kwarg: + name: event.reactor + api_key: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX + reason: {{ data['data']['reason'] }} + action_type: Create +''' +# Import Python libs +from __future__ import absolute_import +import json +import logging +import requests + +# Import Salt libs +import salt.exceptions + +API_ENDPOINT = "https://api.opsgenie.com/v1/json/saltstack?apiKey=" + +log = logging.getLogger(__name__) + + +def post_data(api_key=None, name='OpsGenie Execution Module', reason=None, + action_type=None): + ''' + Post data to OpsGenie. It's designed for Salt's Event Reactor. + + After configuring the sls reaction file as shown above, you can trigger the + module with your designated tag (og-tag in this case). + + CLI Example: + salt-call event.send 'og-tag' '{"reason" : "Overheating CPU!"}' + + Required parameters: + + api_key + It's the API Key you've copied while adding integration in OpsGenie. + + reason + It will be used as alert's default message in OpsGenie. + + action_type + OpsGenie supports the default values Create/Close for action_type. You + can customize this field with OpsGenie's custom actions for other + purposes like adding notes or acknowledging alerts. + + Optional parameters: + + name + It will be used as alert's alias. If you want to use the close + functionality you must provide name field for both states like in + this case. + ''' + if api_key is None or reason is None or action_type is None: + raise salt.exceptions.SaltInvocationError( + 'API Key or Reason or Action Type cannot be None.') + + data = dict() + data['name'] = name + data['reason'] = reason + data['actionType'] = action_type + data['cpuModel'] = __grains__['cpu_model'] + data['cpuArch'] = __grains__['cpuarch'] + data['fqdn'] = __grains__['fqdn'] + data['host'] = __grains__['host'] + data['id'] = __grains__['id'] + data['kernel'] = __grains__['kernel'] + data['kernelRelease'] = __grains__['kernelrelease'] + data['master'] = __grains__['master'] + data['os'] = __grains__['os'] + data['saltPath'] = __grains__['saltpath'] + data['saltVersion'] = __grains__['saltversion'] + data['username'] = __grains__['username'] + data['uuid'] = __grains__['uuid'] + + log.debug('Below data will be posted:\n' + str(data)) + log.debug('API Key:' + api_key + '\t API Endpoint:' + API_ENDPOINT) + + response = requests.post(url=API_ENDPOINT + api_key, data=json.dumps(data), + headers={'Content-Type': 'application/json'}) + return response.status_code, response.text diff --git a/salt/states/opsgenie.py b/salt/states/opsgenie.py new file mode 100644 index 0000000000..d86c32ebb6 --- /dev/null +++ b/salt/states/opsgenie.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- +''' +Create/Close an alert in OpsGenie +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. versionadded:: Oxygen +This state is useful for creating or closing alerts in OpsGenie +during state runs. +.. code-block:: yaml + used_space: + disk.status: + - name: / + - maximum: 79% + - minimum: 20% + + opsgenie_create_action_sender: + opsgenie.create_alert: + - api_key: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX + - reason: 'Disk capacity is out of designated range.' + - name: disk.status + - onfail: + - disk: used_space + + opsgenie_close_action_sender: + opsgenie.close_alert: + - api_key: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX + - name: disk.status + - require: + - disk: used_space + +''' +# Import Python libs +from __future__ import absolute_import +import logging +import inspect + +# Import Salt libs +import salt.exceptions + +log = logging.getLogger(__name__) + + +def create_alert(name=None, api_key=None, reason=None, action_type="Create"): + ''' + Create an alert in OpsGenie. Example usage with Salt's requisites and other + global state arguments could be found above. + + Required Parameters: + + api_key + It's the API Key you've copied while adding integration in OpsGenie. + + reason + It will be used as alert's default message in OpsGenie. + + Optional Parameters: + + name + It will be used as alert's alias. If you want to use the close + functionality you must provide name field for both states like + in above case. + + action_type + OpsGenie supports the default values Create/Close for action_type. + You can customize this field with OpsGenie's custom actions for + other purposes like adding notes or acknowledging alerts. + ''' + + _, _, _, values = inspect.getargvalues(inspect.currentframe()) + log.info("Arguments values:" + str(values)) + + ret = { + 'result': '', + 'name': '', + 'changes': '', + 'comment': '' + } + + if api_key is None or reason is None: + raise salt.exceptions.SaltInvocationError( + 'API Key or Reason cannot be None.') + + if __opts__['test'] is True: + ret[ + 'comment'] = 'Test: {0} alert request will be processed ' \ + 'using the API Key="{1}".'.format( + action_type, + api_key) + + # Return ``None`` when running with ``test=true``. + ret['result'] = None + + return ret + + response_status_code, response_text = __salt__['opsgenie.post_data']( + api_key=api_key, + name=name, + reason=reason, + action_type=action_type + ) + + if 200 <= response_status_code < 300: + log.info( + "POST Request has succeeded with message:" + + response_text + " status code:" + str( + response_status_code)) + ret[ + 'comment'] = 'Test: {0} alert request will be processed' \ + ' using the API Key="{1}".'.format( + action_type, + api_key) + ret['result'] = True + else: + log.error( + "POST Request has failed with error:" + + response_text + " status code:" + str( + response_status_code)) + ret['result'] = False + + return ret + + +def close_alert(name=None, api_key=None, reason="Conditions are met.", + action_type="Close"): + ''' + Close an alert in OpsGenie. It's a wrapper function for create_alert. + Example usage with Salt's requisites and other global state arguments + could be found above. + + Required Parameters: + + name + It will be used as alert's alias. If you want to use the close + functionality you must provide name field for both states like + in above case. + + Optional Parameters: + + api_key + It's the API Key you've copied while adding integration in OpsGenie. + + reason + It will be used as alert's default message in OpsGenie. + + action_type + OpsGenie supports the default values Create/Close for action_type. + You can customize this field with OpsGenie's custom actions for + other purposes like adding notes or acknowledging alerts. + ''' + if name is None: + raise salt.exceptions.SaltInvocationError( + 'Name cannot be None.') + + return create_alert(name, api_key, reason, action_type) From 0cac15e502d1565c718ebb289fb95c6e67dcdf10 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Sat, 30 Sep 2017 22:41:30 +0200 Subject: [PATCH 012/184] Fix to module.run [WIP] DO NOT MERGE For @terminalmage to review --- salt/states/module.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/salt/states/module.py b/salt/states/module.py index 202999e7d8..3c8ce96b50 100644 --- a/salt/states/module.py +++ b/salt/states/module.py @@ -314,22 +314,31 @@ def _call_function(name, returner=None, **kwargs): :return: ''' argspec = salt.utils.args.get_function_argspec(__salt__[name]) + + # func_kw is initialized to a dictinary of keyword arguments the function to be run accepts func_kw = dict(zip(argspec.args[-len(argspec.defaults or []):], # pylint: disable=incompatible-py3-code argspec.defaults or [])) + + # func_args is initialized to a list of keyword arguments the function to be run accepts + func_args = argspec.args[:len(argspec.args or [] ) - len(argspec.defaults or [])] arg_type, na_type, kw_type = [], {}, False for funcset in reversed(kwargs.get('func_args') or []): if not isinstance(funcset, dict): - kw_type = True - if kw_type: - if isinstance(funcset, dict): - arg_type += funcset.values() - na_type.update(funcset) - else: - arg_type.append(funcset) + # We are just receiving a list of args to the function to be run, so just append + # those to the arg list that we will pass to the func. + arg_type.append(funcset) else: - func_kw.update(funcset) + for kwarg_key in funcset.keys(): + # We are going to pass in a keyword argument. The trick here is to make certain + # that if we find that in the *args* list that we pass it there and not as a kwarg + if kwarg_key in func_args: + arg_type.append(funcset[kwarg_key]) + continue + else: + # Otherwise, we're good and just go ahead and pass the keyword/value pair into + # the kwargs list to be run. + func_kw.update(funcset) arg_type.reverse() - _exp_prm = len(argspec.args or []) - len(argspec.defaults or []) _passed_prm = len(arg_type) missing = [] From a6c2d78518bf9e6b9d26cdbd4e245e762ab65def Mon Sep 17 00:00:00 2001 From: Mike Place Date: Sun, 1 Oct 2017 00:39:37 +0200 Subject: [PATCH 013/184] Fix typo found by @s0undt3ch --- salt/states/module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/module.py b/salt/states/module.py index 3c8ce96b50..de701af51c 100644 --- a/salt/states/module.py +++ b/salt/states/module.py @@ -319,7 +319,7 @@ def _call_function(name, returner=None, **kwargs): func_kw = dict(zip(argspec.args[-len(argspec.defaults or []):], # pylint: disable=incompatible-py3-code argspec.defaults or [])) - # func_args is initialized to a list of keyword arguments the function to be run accepts + # func_args is initialized to a list of positional arguments that the function to be run accepts func_args = argspec.args[:len(argspec.args or [] ) - len(argspec.defaults or [])] arg_type, na_type, kw_type = [], {}, False for funcset in reversed(kwargs.get('func_args') or []): From 782e67c199afb51bec10e93fcd65f9af59972247 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Sun, 1 Oct 2017 10:59:09 +0200 Subject: [PATCH 014/184] Lint --- salt/states/module.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/states/module.py b/salt/states/module.py index de701af51c..91e9fb8b57 100644 --- a/salt/states/module.py +++ b/salt/states/module.py @@ -319,8 +319,8 @@ def _call_function(name, returner=None, **kwargs): func_kw = dict(zip(argspec.args[-len(argspec.defaults or []):], # pylint: disable=incompatible-py3-code argspec.defaults or [])) - # func_args is initialized to a list of positional arguments that the function to be run accepts - func_args = argspec.args[:len(argspec.args or [] ) - len(argspec.defaults or [])] + # func_args is initialized to a list of positional arguments that the function to be run accepts + func_args = argspec.args[:len(argspec.args or []) - len(argspec.defaults or [])] arg_type, na_type, kw_type = [], {}, False for funcset in reversed(kwargs.get('func_args') or []): if not isinstance(funcset, dict): From c297ae55575a790637f1dd5d780ce324a027744d Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 29 Sep 2017 17:15:18 -0500 Subject: [PATCH 015/184] Improve failures for module.run states Bare asserts are zero help in troubleshooting. This commit changes the tests that uses bare asserts such that they fail with a useful error mesage as well as the return data from the module.run call. --- tests/unit/states/test_module.py | 79 ++++++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 14 deletions(-) diff --git a/tests/unit/states/test_module.py b/tests/unit/states/test_module.py index 4588e20ca8..2d082fed2a 100644 --- a/tests/unit/states/test_module.py +++ b/tests/unit/states/test_module.py @@ -6,6 +6,7 @@ # Import Python Libs from __future__ import absolute_import from inspect import ArgSpec +import logging # Import Salt Libs import salt.states.module as module @@ -20,6 +21,8 @@ from tests.support.mock import ( patch ) +log = logging.getLogger(__name__) + CMD = 'foo.bar' @@ -91,8 +94,9 @@ class ModuleStateTest(TestCase, LoaderModuleMockMixin): with patch.dict(module.__salt__, {}, clear=True): with patch.dict(module.__opts__, {'use_superseded': ['module.run']}): ret = module.run(**{CMD: None}) - assert ret['comment'] == "Unavailable function: {0}.".format(CMD) - assert not ret['result'] + if ret['comment'] != "Unavailable function: {0}.".format(CMD) \ + or ret['result']: + self.fail('module.run did not fail as expected: {0}'.format(ret)) def test_module_run_hidden_varargs(self): ''' @@ -111,8 +115,9 @@ class ModuleStateTest(TestCase, LoaderModuleMockMixin): ''' with patch.dict(module.__opts__, {'test': True, 'use_superseded': ['module.run']}): ret = module.run(**{CMD: None}) - assert ret['comment'] == "Function {0} to be executed.".format(CMD) - assert ret['result'] + if ret['comment'] != "Function {0} to be executed.".format(CMD) \ + or not ret['result']: + self.fail('module.run failed: {0}'.format(ret)) def test_run_missing_arg(self): ''' @@ -122,7 +127,10 @@ class ModuleStateTest(TestCase, LoaderModuleMockMixin): with patch.dict(module.__salt__, {CMD: _mocked_func_named}): with patch.dict(module.__opts__, {'use_superseded': ['module.run']}): ret = module.run(**{CMD: None}) - assert ret['comment'] == "'{0}' failed: Function expects 1 parameters, got only 0".format(CMD) + expected_comment = \ + "'{0}' failed: Function expects 1 parameters, got only 0".format(CMD) + if ret['comment'] != expected_comment: + self.fail('module.run did not fail as expected: {0}'.format(ret)) def test_run_correct_arg(self): ''' @@ -132,16 +140,17 @@ class ModuleStateTest(TestCase, LoaderModuleMockMixin): with patch.dict(module.__salt__, {CMD: _mocked_func_named}): with patch.dict(module.__opts__, {'use_superseded': ['module.run']}): ret = module.run(**{CMD: ['Fred']}) - assert ret['comment'] == '{0}: Success'.format(CMD) - assert ret['result'] + if ret['comment'] != '{0}: Success'.format(CMD) or not ret['result']: + self.fail('module.run failed: {0}'.format(ret)) def test_run_unexpected_keywords(self): with patch.dict(module.__salt__, {CMD: _mocked_func_args}): with patch.dict(module.__opts__, {'use_superseded': ['module.run']}): ret = module.run(**{CMD: [{'foo': 'bar'}]}) - assert ret['comment'] == "'{0}' failed: {1}() got an unexpected keyword argument " \ - "'foo'".format(CMD, module.__salt__[CMD].__name__) - assert not ret['result'] + expected_comment = "'{0}' failed: {1}() got an unexpected keyword argument " \ + "'foo'".format(CMD, module.__salt__[CMD].__name__) + if ret['comment'] != expected_comment or ret['result']: + self.fail('module.run did not fail as expected: {0}'.format(ret)) def test_run_args(self): ''' @@ -150,7 +159,17 @@ class ModuleStateTest(TestCase, LoaderModuleMockMixin): ''' with patch.dict(module.__salt__, {CMD: _mocked_func_args}): with patch.dict(module.__opts__, {'use_superseded': ['module.run']}): - assert module.run(**{CMD: ['foo', 'bar']})['result'] + try: + ret = module.run(**{CMD: ['foo', 'bar']}) + except Exception as exc: + log.exception('test_run_none_return: raised exception') + self.fail('module.run raised exception: {0}'.format(exc)) + if not ret['result']: + log.exception( + 'test_run_none_return: test failed, result: %s', + ret + ) + self.fail('module.run failed: {0}'.format(ret)) def test_run_none_return(self): ''' @@ -159,7 +178,17 @@ class ModuleStateTest(TestCase, LoaderModuleMockMixin): ''' with patch.dict(module.__salt__, {CMD: _mocked_none_return}): with patch.dict(module.__opts__, {'use_superseded': ['module.run']}): - assert module.run(**{CMD: None})['result'] + try: + ret = module.run(**{CMD: None}) + except Exception as exc: + log.exception('test_run_none_return: raised exception') + self.fail('module.run raised exception: {0}'.format(exc)) + if not ret['result']: + log.exception( + 'test_run_none_return: test failed, result: %s', + ret + ) + self.fail('module.run failed: {0}'.format(ret)) def test_run_typed_return(self): ''' @@ -169,7 +198,18 @@ class ModuleStateTest(TestCase, LoaderModuleMockMixin): for val in [1, 0, 'a', '', (1, 2,), (), [1, 2], [], {'a': 'b'}, {}, True, False]: with patch.dict(module.__salt__, {CMD: _mocked_none_return}): with patch.dict(module.__opts__, {'use_superseded': ['module.run']}): - assert module.run(**{CMD: [{'ret': val}]})['result'] + log.debug('test_run_typed_return: trying %s', val) + try: + ret = module.run(**{CMD: [{'ret': val}]}) + except Exception as exc: + log.exception('test_run_typed_return: raised exception') + self.fail('module.run raised exception: {0}'.format(exc)) + if not ret['result']: + log.exception( + 'test_run_typed_return: test failed, result: %s', + ret + ) + self.fail('module.run failed: {0}'.format(ret)) def test_run_batch_call(self): ''' @@ -182,7 +222,18 @@ class ModuleStateTest(TestCase, LoaderModuleMockMixin): 'second': _mocked_none_return, 'third': _mocked_none_return}, clear=True): for f_name in module.__salt__: - assert module.run(**{f_name: None})['result'] + log.debug('test_run_batch_call: trying %s', f_name) + try: + ret = module.run(**{f_name: None}) + except Exception as exc: + log.exception('test_run_batch_call: raised exception') + self.fail('module.run raised exception: {0}'.format(exc)) + if not ret['result']: + log.exception( + 'test_run_batch_call: test failed, result: %s', + ret + ) + self.fail('module.run failed: {0}'.format(ret)) def test_module_run_module_not_available(self): ''' From e21d8e9583bba816b6ebc8c9bed379fb4790dfa4 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 2 Oct 2017 11:40:00 -0500 Subject: [PATCH 016/184] Use six.iterkeys() instead of dict.keys() This avoids generating a new list on PY2. --- salt/states/module.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/states/module.py b/salt/states/module.py index 91e9fb8b57..2e907959f7 100644 --- a/salt/states/module.py +++ b/salt/states/module.py @@ -178,6 +178,7 @@ from __future__ import absolute_import import salt.loader import salt.utils import salt.utils.jid +from salt.ext import six from salt.ext.six.moves import range from salt.ext.six.moves import zip from salt.exceptions import SaltInvocationError @@ -315,7 +316,7 @@ def _call_function(name, returner=None, **kwargs): ''' argspec = salt.utils.args.get_function_argspec(__salt__[name]) - # func_kw is initialized to a dictinary of keyword arguments the function to be run accepts + # func_kw is initialized to a dictionary of keyword arguments the function to be run accepts func_kw = dict(zip(argspec.args[-len(argspec.defaults or []):], # pylint: disable=incompatible-py3-code argspec.defaults or [])) @@ -328,7 +329,7 @@ def _call_function(name, returner=None, **kwargs): # those to the arg list that we will pass to the func. arg_type.append(funcset) else: - for kwarg_key in funcset.keys(): + for kwarg_key in six.iterkeys(funcset): # We are going to pass in a keyword argument. The trick here is to make certain # that if we find that in the *args* list that we pass it there and not as a kwarg if kwarg_key in func_args: From 2337904656e5b935ff330bf1cee5a54299439960 Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 2 Oct 2017 17:18:30 -0400 Subject: [PATCH 017/184] Add updated release notes to 2017.7.2 branch --- doc/topics/releases/2017.7.2.rst | 3162 ++++++++++++++++++++++++++++++ 1 file changed, 3162 insertions(+) create mode 100644 doc/topics/releases/2017.7.2.rst diff --git a/doc/topics/releases/2017.7.2.rst b/doc/topics/releases/2017.7.2.rst new file mode 100644 index 0000000000..ac8ef25146 --- /dev/null +++ b/doc/topics/releases/2017.7.2.rst @@ -0,0 +1,3162 @@ +=========================== +Salt 2017.7.2 Release Notes +=========================== + + +Changes for v2017.7.1..v2017.7.2 +-------------------------------- + +Extended changelog courtesy of Todd Stansell (https://github.com/tjstansell/salt-changelogs): + +*Generated at: 2017-10-02T21:10:14Z* + +Statistics +========== + +- Total Merges: **328** +- Total Issue references: **134** +- Total PR references: **391** + +Changes +======= + +- **PR** `#43868`_: (*rallytime*) Back-port `#43847`_ to 2017.7.2 + * Fix to module.run + +- **PR** `#43756`_: (*gtmanfred*) split build and install for pkg osx + @ *2017-09-26T20:51:28Z* + + * 88414d5 Merge pull request `#43756`_ from gtmanfred/2017.7.2 + * f7df41f split build and install for pkg osx + +- **PR** `#43585`_: (*rallytime*) Back-port `#43330`_ to 2017.7.2 + @ *2017-09-19T17:33:34Z* + + - **ISSUE** `#43077`_: (*Manoj2087*) Issue with deleting key via wheel + | refs: `#43330`_ + - **PR** `#43330`_: (*terminalmage*) Fix reactor regression + unify reactor config schema + | refs: `#43585`_ + * 89f6292 Merge pull request `#43585`_ from rallytime/`bp-43330`_ + * c4f693b Merge branch '2017.7.2' into `bp-43330`_ + +- **PR** `#43586`_: (*rallytime*) Back-port `#43526`_ to 2017.7.2 + @ *2017-09-19T15:36:27Z* + + - **ISSUE** `#43447`_: (*UtahDave*) When using Syndic with Multi Master the top level master doesn't reliably get returns from lower minion. + | refs: `#43526`_ + - **PR** `#43526`_: (*DmitryKuzmenko*) Forward events to all masters syndic connected to + | refs: `#43586`_ + * abb7fe4 Merge pull request `#43586`_ from rallytime/`bp-43526`_ + * e076e9b Forward events to all masters syndic connected to. + + * 7abd07f Simplify client logic + + * b5f1069 Improve the reactor documentation + + * 7a2f12b Include a better example for reactor in master conf file + + * 531cac6 Rewrite the reactor unit tests + + * 2a35ab7 Unify reactor configuration, fix caller reactors + + * 4afb179 Un-deprecate passing kwargs outside of 'kwarg' param + +- **PR** `#43551`_: (*twangboy*) Fix preinstall script on OSX for 2017.7.2 + @ *2017-09-18T18:35:35Z* + + * 3d3b093 Merge pull request `#43551`_ from twangboy/osx_fix_preinstall_2017.7.2 + * c3d9fb6 Merge branch '2017.7.2' into osx_fix_preinstall_2017.7.2 + +- **PR** `#43509`_: (*rallytime*) Back-port `#43333`_ to 2017.7.2 + @ *2017-09-15T21:21:40Z* + + - **ISSUE** `#2`_: (*thatch45*) salt job queries + - **PR** `#43333`_: (*damon-atkins*) Docs are wrong cache_dir (bool) and cache_file (str) cannot be passed as params + 1 bug + | refs: `#43509`_ + * 24691da Merge pull request `#43509`_ from rallytime/`bp-43333`_-2017.7.2 + * b3dbafb Update doco + + * 5cdcdbf Update win_pkg.py + + * c3e1666 Docs are wrong cache_dir (bool) and cache_file (str) cannot be passed on the cli (`#2`_) + + * f33395f Fix logic in `/etc/paths.d/salt` detection + +- **PR** `#43440`_: (*rallytime*) Back-port `#43421`_ to 2017.7.2 + @ *2017-09-11T20:59:53Z* + + - **PR** `#43421`_: (*gtmanfred*) Revert "Reduce fileclient.get_file latency by merging _file_find and … + | refs: `#43440`_ + * 8964cac Merge pull request `#43440`_ from rallytime/`bp-43421`_ + * ea6e661 Revert "Reduce fileclient.get_file latency by merging _file_find and _file_hash" + +- **PR** `#43377`_: (*rallytime*) Back-port `#43193`_ to 2017.7.2 + @ *2017-09-11T15:32:23Z* + + - **PR** `#43193`_: (*jettero*) Prevent spurious "Template does not exist" error + | refs: `#43377`_ + - **PR** `#39516`_: (*jettero*) Prevent spurious "Template does not exist" error + | refs: `#43193`_ + * 7fda186 Merge pull request `#43377`_ from rallytime/`bp-43193`_ + * 842b07f Prevent spurious "Template does not exist" error + +- **PR** `#43315`_: (*rallytime*) Back-port `#43283`_ to 2017.7.2 + @ *2017-09-05T20:04:25Z* + + - **ISSUE** `#42459`_: (*iavael*) Broken ldap groups retrieval in salt.auth.ldap after upgrade to 2017.7 + | refs: `#43283`_ + - **PR** `#43283`_: (*DmitryKuzmenko*) Fix ldap token groups auth. + | refs: `#43315`_ + * 85dba1e Merge pull request `#43315`_ from rallytime/`bp-43283`_ + * f29f5b0 Fix for tests: don't require 'groups' in the eauth token. + + * 56938d5 Fix ldap token groups auth. + +- **PR** `#43266`_: (*gtmanfred*) switch virtualbox cloud driver to use __utils__ + @ *2017-08-30T18:36:20Z* + + - **ISSUE** `#43259`_: (*mahesh21*) NameError: global name '__opts__' is not defined + | refs: `#43266`_ + * 26ff808 Merge pull request `#43266`_ from gtmanfred/virtualbox + * 382bf92 switch virtualbox cloud driver to use __utils__ + +- **PR** `#43073`_: (*Mapel88*) Fix bug `#42936`_ - win_iis module container settings + @ *2017-08-30T18:34:37Z* + + - **ISSUE** `#43110`_: (*Mapel88*) bug in iis_module - create_cert_binding + - **ISSUE** `#42936`_: (*Mapel88*) bug in win_iis module & state - container_setting + | refs: `#43073`_ + * ee209b1 Merge pull request `#43073`_ from Mapel88/patch-2 + * b1a3d15 Remove trailing whitespace for linter + + * 25c8190 Fix pylint errors + + * 1eba8c4 Fix pylint errors + + * 290d7b5 Fix plint errors + + * f4f3242 Fix plint errors + + * ec20e9a Fix bug `#43110`_ - win_iis module + + * 009ef66 Fix dictionary keys from string to int + + * dc793f9 Fix bug `#42936`_ - win_iis state + + * 13404a4 Fix bug `#42936`_ - win_iis module + +- **PR** `#43254`_: (*twangboy*) Fix `unit.modules.test_inspect_collector` on Windows + @ *2017-08-30T15:46:07Z* + + * ec1bedc Merge pull request `#43254`_ from twangboy/win_fix_test_inspect_collector + * b401340 Fix `unit.modules.test_inspect_collector` on Windows + +- **PR** `#43255`_: (*gtmanfred*) always return a dict object + @ *2017-08-30T14:47:15Z* + + - **ISSUE** `#43241`_: (*mirceaulinic*) Error whilst collecting napalm grains + | refs: `#43255`_ + * 1fc7307 Merge pull request `#43255`_ from gtmanfred/2017.7 + * 83b0bab opt_args needs to be a dict + +- **PR** `#43229`_: (*twangboy*) Bring changes from `#43228`_ to 2017.7 + @ *2017-08-30T14:26:55Z* + + - **PR** `#43228`_: (*twangboy*) Win fix pkg.install + | refs: `#43229`_ + * fa904ee Merge pull request `#43229`_ from twangboy/win_fix_pkg.install-2017.7 + * e007a1c Fix regex, add `.` + + * 23ec47c Add _ to regex search + + * b1788b1 Bring changes from `#43228`_ to 2017.7 + +- **PR** `#43251`_: (*twangboy*) Skips `unit.modules.test_groupadd` on Windows + @ *2017-08-30T13:56:36Z* + + * 25666f8 Merge pull request `#43251`_ from twangboy/win_skip_test_groupadd + * 5185071 Skips `unit.modules.test_groupadd` on Windows + +- **PR** `#43256`_: (*twangboy*) Skip mac tests for user and group + @ *2017-08-30T13:18:13Z* + + * a8e0962 Merge pull request `#43256`_ from twangboy/win_skip_mac_tests + * cec627a Skip mac tests for user and group + +- **PR** `#43226`_: (*lomeroe*) Fixes for issues in PR `#43166`_ + @ *2017-08-29T19:05:39Z* + + - **ISSUE** `#42279`_: (*dafyddj*) win_lgpo matches multiple policies due to startswith() + | refs: `#43116`_ `#43116`_ `#43166`_ `#43226`_ `#43156`_ + - **PR** `#43166`_: (*lomeroe*) Backport `#43116`_ to 2017.7 + | refs: `#43226`_ + - **PR** `#43156`_: (*lomeroe*) Backport `#43116`_ to 2017.7 + | refs: `#43166`_ + - **PR** `#43116`_: (*lomeroe*) Fix 42279 in develop + | refs: `#43166`_ `#43156`_ + - **PR** `#39773`_: (*twangboy*) Make win_file use the win_dacl salt util + | refs: `#43226`_ + * ac2189c Merge pull request `#43226`_ from lomeroe/fix_43166 + * 0c424dc Merge branch '2017.7' into fix_43166 + + * 324cfd8d correcting bad format statement in search for policy to be disabled (fix for `#43166`_) verify that file exists before attempting to remove (fix for commits from `#39773`_) + +- **PR** `#43227`_: (*twangboy*) Fix `unit.fileserver.test_gitfs` for Windows + @ *2017-08-29T19:03:36Z* + + * 6199fb4 Merge pull request `#43227`_ from twangboy/win_fix_unit_test_gitfs + * c956d24 Fix is_windows detection when USERNAME missing + + * 869e8cc Fix `unit.fileserver.test_gitfs` for Windows + +- **PR** `#43217`_: (*rallytime*) [2017.7] Merge forward from 2016.11 to 2017.7 + @ *2017-08-28T16:36:28Z* + + - **ISSUE** `#43101`_: (*aogier*) genesis.bootstrap fails if no pkg AND exclude_pkgs (which can't be a string) + | refs: `#43103`_ + - **ISSUE** `#42642`_: (*githubcdr*) state.augeas + | refs: `#42669`_ `#43202`_ + - **ISSUE** `#42329`_: (*jagguli*) State git.latest does not pull latest tags + | refs: `#42663`_ + - **PR** `#43202`_: (*garethgreenaway*) Reverting previous augeas module changes + - **PR** `#43103`_: (*aogier*) genesis.bootstrap deboostrap fix + - **PR** `#42663`_: (*jagguli*) Check remote tags before deciding to do a fetch `#42329`_ + * 6adc03e Merge pull request `#43217`_ from rallytime/merge-2017.7 + * 3911df2 Merge branch '2016.11' into '2017.7' + + * 5308c27 Merge pull request `#43202`_ from garethgreenaway/42642_2016_11_augeas_module_revert_fix + + * ef7e93e Reverting this change due to it breaking other uses. + + * f16b724 Merge pull request `#43103`_ from aogier/43101-genesis-bootstrap + + * db94f3b better formatting + + * e5cc667 tests: fix a leftover and simplify some parts + + * 13e5997 lint + + * 216ced6 allow comma-separated pkgs lists, quote args, test deb behaviour + + * d8612ae fix debootstrap and enhance packages selection/deletion via cmdline + + * 4863771 Merge pull request `#42663`_ from StreetHawkInc/fix_git_tag_check + + * 2b5af5b Remove refs/tags prefix from remote tags + + * 3f2e96e Convert set to list for serializer + + * 2728e5d Only include new tags in changes + + * 4b1df2f Exclude annotated tags from checks + + * 389c037 Check remote tags before deciding to do a fetch `#42329`_ + +- **PR** `#43201`_: (*rallytime*) [2017.7] Merge forward from 2016.11 to 2017.7 + @ *2017-08-25T22:56:46Z* + + - **ISSUE** `#43198`_: (*corywright*) disk.format_ needs to be aliased to disk.format + | refs: `#43199`_ + - **ISSUE** `#43143`_: (*abulford*) git.detached does not fetch if rev is missing from local + | refs: `#43178`_ + - **ISSUE** `#495`_: (*syphernl*) mysql.* without having MySQL installed/configured gives traceback + | refs: `#43196`_ + - **PR** `#43199`_: (*corywright*) Add `disk.format` alias for `disk.format_` + - **PR** `#43196`_: (*gtmanfred*) Pin request install to version for npm tests + - **PR** `#43179`_: (*terminalmage*) Fix missed deprecation + - **PR** `#43178`_: (*terminalmage*) git.detached: Fix traceback when rev is a SHA and is not present locally + - **PR** `#43173`_: (*Ch3LL*) Add New Release Branch Strategy to Contribution Docs + - **PR** `#43171`_: (*terminalmage*) Add warning about adding new functions to salt/utils/__init__.py + * a563a94 Merge pull request `#43201`_ from rallytime/merge-2017.7 + * d40eba6 Merge branch '2016.11' into '2017.7' + + * 4193e7f Merge pull request `#43199`_ from corywright/disk-format-alias + + * f00d3a9 Add `disk.format` alias for `disk.format_` + + * 5471f9f Merge pull request `#43196`_ from gtmanfred/2016.11 + + * ccd2241 Pin request install to version + + * ace2715 Merge pull request `#43178`_ from terminalmage/issue43143 + + * 2640833 git.detached: Fix traceback when rev is a SHA and is not present locally + + * 12e9507 Merge pull request `#43179`_ from terminalmage/old-deprecation + + * 3adf8ad Fix missed deprecation + + * b595440 Merge pull request `#43171`_ from terminalmage/salt-utils-warning + + * 7b5943a Add warning about adding new functions to salt/utils/__init__.py + + * 4f273ca Merge pull request `#43173`_ from Ch3LL/add_branch_docs + + * 1b24244 Add New Release Branch Strategy to Contribution Docs + +- **PR** `#42997`_: (*twangboy*) Fix `unit.test_test_module_names` for Windows + @ *2017-08-25T21:19:11Z* + + * ce04ab4 Merge pull request `#42997`_ from twangboy/win_fix_test_module_names + * 2722e95 Use os.path.join to create paths + +- **PR** `#43006`_: (*SuperPommeDeTerre*) Try to fix `#26995`_ + @ *2017-08-25T21:16:07Z* + + - **ISSUE** `#26995`_: (*jbouse*) Issue with artifactory.downloaded and snapshot artifacts + | refs: `#43006`_ `#43006`_ + * c0279e4 Merge pull request `#43006`_ from SuperPommeDeTerre/SuperPommeDeTerre-patch-`#26995`_ + * 30dd6f5 Merge remote-tracking branch 'upstream/2017.7' into SuperPommeDeTerre-patch-`#26995`_ + + * f42ae9b Merge branch 'SuperPommeDeTerre-patch-`#26995`_' of https://github.com/SuperPommeDeTerre/salt into SuperPommeDeTerre-patch-`#26995`_ + + * 50ee3d5 Merge remote-tracking branch 'remotes/origin/2017.7' into SuperPommeDeTerre-patch-`#26995`_ + + * 0b666e1 Fix typo. + + * 1b8729b Fix for `#26995`_ + + * e314102 Fix typo. + + * db11e19 Fix for `#26995`_ + +- **PR** `#43184`_: (*terminalmage*) docker.compare_container: Perform boolean comparison when one side's value is null/None + @ *2017-08-25T18:42:11Z* + + - **ISSUE** `#43162`_: (*MorphBonehunter*) docker_container.running interference with restart_policy + | refs: `#43184`_ + * b6c5314 Merge pull request `#43184`_ from terminalmage/issue43162 + * 081f42a docker.compare_container: Perform boolean comparison when one side's value is null/None + +- **PR** `#43165`_: (*mirceaulinic*) Improve napalm state output in debug mode + @ *2017-08-24T23:05:37Z* + + * 688125b Merge pull request `#43165`_ from cloudflare/fix-napalm-ret + * c10717d Lint and fix + + * 1cd33cb Simplify the loaded_ret logic + + * 0bbea6b Document the new compliance_report arg + + * 3a90610 Include compliance reports + + * 3634055 Improve napalm state output in debug mode + +- **PR** `#43155`_: (*terminalmage*) Resolve image ID during container comparison + @ *2017-08-24T22:09:47Z* + + * a6a327b Merge pull request `#43155`_ from terminalmage/issue43001 + * 0186835 Fix docstring in test + + * a0bb654 Fixing lint issues + + * d5b2a0b Resolve image ID during container comparison + +- **PR** `#43170`_: (*rallytime*) [2017.7] Merge forward from 2016.11 to 2017.7 + @ *2017-08-24T19:22:26Z* + + - **PR** `#43151`_: (*ushmodin*) state.sls hangs on file.recurse with clean: True on windows + - **PR** `#42969`_: (*ushmodin*) state.sls hangs on file.recurse with clean: True on windows + | refs: `#43151`_ + * c071fd4 Merge pull request `#43170`_ from rallytime/merge-2017.7 + * 3daad5a Merge branch '2016.11' into '2017.7' + + * 669b376 Merge pull request `#43151`_ from ushmodin/2016.11 + + * c5841e2 state.sls hangs on file.recurse with clean: True on windows + +- **PR** `#43168`_: (*rallytime*) Back-port `#43041`_ to 2017.7 + @ *2017-08-24T19:07:23Z* + + - **ISSUE** `#43040`_: (*darcoli*) gitFS ext_pillar with branch name __env__ results in empty pillars + | refs: `#43041`_ `#43041`_ + - **PR** `#43041`_: (*darcoli*) Do not try to match pillarenv with __env__ + | refs: `#43168`_ + * 034c325 Merge pull request `#43168`_ from rallytime/`bp-43041`_ + * d010b74 Do not try to match pillarenv with __env__ + +- **PR** `#43172`_: (*rallytime*) Move new utils/__init__.py funcs to utils.files.py + @ *2017-08-24T19:05:30Z* + + - **PR** `#43056`_: (*damon-atkins*) safe_filename_leaf(file_basename) and safe_filepath(file_path_name) + | refs: `#43172`_ + * d48938e Merge pull request `#43172`_ from rallytime/move-utils-funcs + * 5385c79 Move new utils/__init__.py funcs to utils.files.py + +- **PR** `#43061`_: (*pabloh007*) Have docker.save use the image name when valid if not use image id, i… + @ *2017-08-24T16:32:02Z* + + - **ISSUE** `#43043`_: (*pabloh007*) docker.save and docker.load problem + | refs: `#43061`_ `#43061`_ + * e60f586 Merge pull request `#43061`_ from pabloh007/fix-save-image-name-id + * 0ffc57d Have docker.save use the image name when valid if not use image id, issue when loading and image is savid with id issue `#43043`_ + +- **PR** `#43166`_: (*lomeroe*) Backport `#43116`_ to 2017.7 + | refs: `#43226`_ + @ *2017-08-24T15:01:23Z* + + - **ISSUE** `#42279`_: (*dafyddj*) win_lgpo matches multiple policies due to startswith() + | refs: `#43116`_ `#43116`_ `#43166`_ `#43226`_ `#43156`_ + - **PR** `#43156`_: (*lomeroe*) Backport `#43116`_ to 2017.7 + | refs: `#43166`_ + - **PR** `#43116`_: (*lomeroe*) Fix 42279 in develop + | refs: `#43166`_ `#43156`_ + * 9da5754 Merge pull request `#43166`_ from lomeroe/`bp-43116`_-2017.7 + * af181b3 correct fopen calls from salt.utils for 2017.7 + + * f74480f lint fix + + * ecd446f track xml namespace to ensure policies w/duplicate IDs or Names do not conflict + + * 9f3047c add additional checks for ADM policies that have the same ADMX policy ID (`#42279`_) + +- **PR** `#43056`_: (*damon-atkins*) safe_filename_leaf(file_basename) and safe_filepath(file_path_name) + | refs: `#43172`_ + @ *2017-08-23T17:35:02Z* + + * 44b3cae Merge pull request `#43056`_ from damon-atkins/2017.7 + * 08ded15 more lint + + * 6e9c095 fix typo + + * ee41171 lint fixes + + * 8c864f0 fix missing imports + + * 964cebd safe_filename_leaf(file_basename) and safe_filepath(file_path_name) + +- **PR** `#43146`_: (*rallytime*) [2017.7] Merge forward from 2016.11 to 2017.7 + @ *2017-08-23T16:56:10Z* + + - **ISSUE** `#43036`_: (*mcarlton00*) Linux VMs in Bhyve aren't displayed properly in grains + | refs: `#43037`_ + - **PR** `#43100`_: (*vutny*) [DOCS] Add missing `utils` sub-dir listed for `extension_modules` + - **PR** `#43037`_: (*mcarlton00*) Issue `#43036`_ Bhyve virtual grain in Linux VMs + - **PR** `#42986`_: (*renner*) Notify systemd synchronously (via NOTIFY_SOCKET) + * 6ca9131 Merge pull request `#43146`_ from rallytime/merge-2017.7 + * bcbe180 Merge branch '2016.11' into '2017.7' + + * ae9d2b7 Merge pull request `#42986`_ from renner/systemd-notify + + * 79c53f3 Fallback to systemd_notify_call() in case of socket.error + + * f176547 Notify systemd synchronously (via NOTIFY_SOCKET) + + * b420fbe Merge pull request `#43037`_ from mcarlton00/fix-bhyve-grains + + * 73315f0 Issue `#43036`_ Bhyve virtual grain in Linux VMs + + * 0a86f2d Merge pull request `#43100`_ from vutny/doc-add-missing-utils-ext + + * af743ff [DOCS] Add missing `utils` sub-dir listed for `extension_modules` + +- **PR** `#43123`_: (*twangboy*) Fix `unit.utils.test_which` for Windows + @ *2017-08-23T16:01:39Z* + + * 03f6521 Merge pull request `#43123`_ from twangboy/win_fix_test_which + * ed97cff Fix `unit.utils.test_which` for Windows + +- **PR** `#43142`_: (*rallytime*) Back-port `#43068`_ to 2017.7 + @ *2017-08-23T15:56:48Z* + + - **ISSUE** `#42505`_: (*ikogan*) selinux.fcontext_policy_present exception looking for selinux.filetype_id_to_string + | refs: `#43068`_ + - **PR** `#43068`_: (*ixs*) Mark selinux._filetype_id_to_string as public function + | refs: `#43142`_ + * 5a4fc07 Merge pull request `#43142`_ from rallytime/`bp-43068`_ + * efc1c8c Mark selinux._filetype_id_to_string as public function + +- **PR** `#43038`_: (*twangboy*) Fix `unit.utils.test_url` for Windows + @ *2017-08-23T13:35:25Z* + + * 0467a0e Merge pull request `#43038`_ from twangboy/win_unit_utils_test_url + * 7f5ee55 Fix `unit.utils.test_url` for Windows + +- **PR** `#43097`_: (*twangboy*) Fix `group.present` for Windows + @ *2017-08-23T13:19:56Z* + + * e9ccaa6 Merge pull request `#43097`_ from twangboy/win_fix_group + * 43b0360 Fix lint + + * 9ffe315 Add kwargs + + * 4f4e34c Fix group state for Windows + +- **PR** `#43115`_: (*rallytime*) Back-port `#42067`_ to 2017.7 + @ *2017-08-22T20:09:52Z* + + - **PR** `#42067`_: (*vitaliyf*) Removed several uses of name.split('.')[0] in SoftLayer driver. + | refs: `#43115`_ + * 8140855 Merge pull request `#43115`_ from rallytime/`bp-42067`_ + * 8a6ad0a Fixed typo. + + * 9a5ae2b Removed several uses of name.split('.')[0] in SoftLayer driver. + +- **PR** `#42962`_: (*twangboy*) Fix `unit.test_doc test` for Windows + @ *2017-08-22T18:06:23Z* + + * 1e1a810 Merge pull request `#42962`_ from twangboy/win_unit_test_doc + * 201ceae Fix lint, remove debug statement + + * 37029c1 Fix unit.test_doc test + +- **PR** `#42995`_: (*twangboy*) Fix malformed requisite for Windows + @ *2017-08-22T16:50:01Z* + + * d347d1c Merge pull request `#42995`_ from twangboy/win_fix_invalid_requisite + * 93390de Fix malformed requisite for Windows + +- **PR** `#43108`_: (*rallytime*) Back-port `#42988`_ to 2017.7 + @ *2017-08-22T16:49:27Z* + + - **PR** `#42988`_: (*thusoy*) Fix broken negation in iptables + | refs: `#43108`_ + * 1c7992a Merge pull request `#43108`_ from rallytime/`bp-42988`_ + * 1a987cb Fix broken negation in iptables + +- **PR** `#43107`_: (*rallytime*) [2017.7] Merge forward from 2016.11 to 2017.7 + @ *2017-08-22T16:11:25Z* + + - **ISSUE** `#42869`_: (*abednarik*) Git Module : Failed to update repository + | refs: `#43064`_ + - **ISSUE** `#42041`_: (*lorengordon*) pkg.list_repo_pkgs fails to find pkgs with spaces around yum repo enabled value + | refs: `#43054`_ + - **ISSUE** `#15171`_: (*JensRantil*) Maximum recursion limit hit related to requisites + | refs: `#42985`_ + - **PR** `#43092`_: (*blarghmatey*) Fixed issue with silently passing all tests in Testinfra module + - **PR** `#43064`_: (*terminalmage*) Fix race condition in git.latest + - **PR** `#43060`_: (*twangboy*) Osx update pkg scripts + - **PR** `#43054`_: (*lorengordon*) Uses ConfigParser to read yum config files + - **PR** `#42985`_: (*DmitryKuzmenko*) Properly handle `prereq` having lost requisites. + - **PR** `#42045`_: (*arount*) Fix: salt.modules.yumpkg: ConfigParser to read ini like files. + | refs: `#43054`_ + * c6993f4 Merge pull request `#43107`_ from rallytime/merge-2017.7 + * 328dd6a Merge branch '2016.11' into '2017.7' + + * e2bf2f4 Merge pull request `#42985`_ from DSRCorporation/bugs/15171_recursion_limit + + * 651b1ba Properly handle `prereq` having lost requisites. + + * e513333 Merge pull request `#43092`_ from mitodl/2016.11 + + * d4b113a Fixed issue with silently passing all tests in Testinfra module + + * 77a443c Merge pull request `#43060`_ from twangboy/osx_update_pkg_scripts + + * ef8a14c Remove /opt/salt instead of /opt/salt/bin + + * 2dd62aa Add more information to the description + + * f44f5b7 Only stop services if they are running + + * 3b62bf9 Remove salt from the path + + * ebdca3a Update pkg-scripts + + * 1b1b6da Merge pull request `#43064`_ from terminalmage/issue42869 + + * 093c0c2 Fix race condition in git.latest + + * 96e8e83 Merge pull request `#43054`_ from lorengordon/fix/yumpkg/config-parser + + * 3b2cb81 fix typo in salt.modules.yumpkg + + * 38add0e break if leading comments are all fetched + + * d7f65dc fix configparser import & log if error was raised + + * ca1b1bb use configparser to parse yum repo file + +- **PR** `#42996`_: (*twangboy*) Fix `unit.test_stateconf` for Windows + @ *2017-08-21T22:43:58Z* + + * f9b4976 Merge pull request `#42996`_ from twangboy/win_fix_test_stateconf + * 92dc3c0 Use os.sep for path + +- **PR** `#43024`_: (*twangboy*) Fix `unit.utils.test_find` for Windows + @ *2017-08-21T22:38:10Z* + + * 19fc644 Merge pull request `#43024`_ from twangboy/win_unit_utils_test_find + * fbe54c9 Remove unused import six (lint) + + * b04d1a2 Fix `unit.utils.test_find` for Windows + +- **PR** `#43088`_: (*gtmanfred*) allow docker util to be reloaded with reload_modules + @ *2017-08-21T22:14:37Z* + + * 1a53116 Merge pull request `#43088`_ from gtmanfred/2017.7 + * 373a9a0 allow docker util to be reloaded with reload_modules + +- **PR** `#43091`_: (*blarghmatey*) Fixed issue with silently passing all tests in Testinfra module + @ *2017-08-21T22:06:22Z* + + * 83e528f Merge pull request `#43091`_ from mitodl/2017.7 + * b502560 Fixed issue with silently passing all tests in Testinfra module + +- **PR** `#41994`_: (*twangboy*) Fix `unit.modules.test_cmdmod` on Windows + @ *2017-08-21T21:53:01Z* + + * 5482524 Merge pull request `#41994`_ from twangboy/win_unit_test_cmdmod + * a5f7288 Skip test that uses pwd, not available on Windows + +- **PR** `#42933`_: (*garethgreenaway*) Fixes to osquery module + @ *2017-08-21T20:48:31Z* + + - **ISSUE** `#42873`_: (*TheVakman*) osquery Data Empty Upon Return / Reporting Not Installed + | refs: `#42933`_ + * b33c4ab Merge pull request `#42933`_ from garethgreenaway/42873_2017_7_osquery_fix + * 8915e62 Removing an import that is not needed. + + * 74bc377 Updating the other function that uses cmd.run_all + + * e6a4619 Better approach without using python_shell=True. + + * 5ac41f4 When running osquery commands through cmd.run we should pass python_shell=True to ensure everything is formatted right. `#42873`_ + +- **PR** `#43093`_: (*gtmanfred*) Fix ec2 list_nodes_full to work on 2017.7 + @ *2017-08-21T20:21:21Z* + + * 53c2115 Merge pull request `#43093`_ from gtmanfred/ec2 + * c7cffb5 This block isn't necessary + + * b7283bc _vm_provider_driver isn't needed anymore + +- **PR** `#43087`_: (*rallytime*) Back-port `#42174`_ to 2017.7 + @ *2017-08-21T18:40:18Z* + + - **ISSUE** `#43085`_: (*brejoc*) Patch for Kubernetes module missing from 2017.7 and 2017.7.1 + | refs: `#43087`_ + - **PR** `#42174`_: (*mcalmer*) kubernetes: provide client certificate authentication + | refs: `#43087`_ + * 32f9ade Merge pull request `#43087`_ from rallytime/`bp-42174`_ + * cf65636 add support for certificate authentication to kubernetes module + +- **PR** `#43029`_: (*terminalmage*) Normalize the salt caching API + @ *2017-08-21T16:54:58Z* + + * 882fcd8 Merge pull request `#43029`_ from terminalmage/fix-func-alias + * f8f74a3 Update localfs cache tests to reflect changes to func naming + + * c4ae79b Rename other refs to cache.ls with cache.list + + * ee59d12 Normalize the salt caching API + +- **PR** `#43039`_: (*gtmanfred*) catch ImportError for kubernetes.client import + @ *2017-08-21T14:32:38Z* + + - **ISSUE** `#42843`_: (*brejoc*) Kubernetes module won't work with Kubernetes Python client > 1.0.2 + | refs: `#42845`_ + - **PR** `#42845`_: (*brejoc*) API changes for Kubernetes version 2.0.0 + | refs: `#43039`_ + * dbee735 Merge pull request `#43039`_ from gtmanfred/kube + * 7e269cb catch ImportError for kubernetes.client import + +- **PR** `#43058`_: (*rallytime*) Update release version number for jenkins.run function + @ *2017-08-21T14:13:34Z* + + * c56a849 Merge pull request `#43058`_ from rallytime/fix-release-num + * d7eef70 Update release version number for jenkins.run function + +- **PR** `#43051`_: (*rallytime*) [2017.7] Merge forward from 2016.11 to 2017.7 + @ *2017-08-18T17:05:57Z* + + - **ISSUE** `#42992`_: (*pabloh007*) docker.save flag push does is ignored + - **ISSUE** `#42627`_: (*taigrrr8*) salt-cp no longer works. Was working a few months back. + | refs: `#42890`_ + - **ISSUE** `#40490`_: (*alxwr*) saltstack x509 incompatible to m2crypto 0.26.0 + | refs: `#42760`_ + - **PR** `#43048`_: (*rallytime*) Back-port `#43031`_ to 2016.11 + - **PR** `#43033`_: (*rallytime*) Back-port `#42760`_ to 2016.11 + - **PR** `#43032`_: (*rallytime*) Back-port `#42547`_ to 2016.11 + - **PR** `#43031`_: (*gtmanfred*) use a ruby gem that doesn't have dependencies + | refs: `#43048`_ + - **PR** `#43027`_: (*pabloh007*) Fixes ignore push flag for docker.push module issue `#42992`_ + - **PR** `#43026`_: (*rallytime*) Back-port `#43020`_ to 2016.11 + - **PR** `#43023`_: (*terminalmage*) Fixes/improvements to Jenkins state/module + - **PR** `#43021`_: (*terminalmage*) Use socket.AF_INET6 to get the correct value instead of doing an OS check + - **PR** `#43020`_: (*gtmanfred*) test with gem that appears to be abandoned + | refs: `#43026`_ + - **PR** `#43019`_: (*rallytime*) Update bootstrap script to latest stable: v2017.08.17 + - **PR** `#43014`_: (*Ch3LL*) Change AF_INET6 family for mac in test_host_to_ips + | refs: `#43021`_ + - **PR** `#43009`_: (*rallytime*) [2016.11] Merge forward from 2016.3 to 2016.11 + - **PR** `#42954`_: (*Ch3LL*) [2016.3] Bump latest and previous versions + - **PR** `#42949`_: (*Ch3LL*) Add Security Notice to 2016.3.7 Release Notes + - **PR** `#42942`_: (*Ch3LL*) [2016.3] Add clean_id function to salt.utils.verify.py + - **PR** `#42890`_: (*DmitryKuzmenko*) Make chunked mode in salt-cp optional + - **PR** `#42760`_: (*AFriemann*) Catch TypeError thrown by m2crypto when parsing missing subjects in c… + | refs: `#43033`_ + - **PR** `#42547`_: (*blarghmatey*) Updated testinfra modules to work with more recent versions + | refs: `#43032`_ + * 7b0c947 Merge pull request `#43051`_ from rallytime/merge-2017.7 + * 153a463 Lint: Add missing blank line + + * 84829a6 Merge branch '2016.11' into '2017.7' + + * 43aa46f Merge pull request `#43048`_ from rallytime/`bp-43031`_ + + * 35e4504 use a ruby gem that doesn't have dependencies + + * ad89ff3 Merge pull request `#43023`_ from terminalmage/fix-jenkins-xml-caching + + * 33fd8ff Update jenkins.py + + * fc306fc Add missing colon in `if` statement + + * 822eabc Catch exceptions raised when making changes to jenkins + + * 91b583b Improve and correct execption raising + + * f096917 Raise an exception if we fail to cache the config xml + + * 2957467 Merge pull request `#43026`_ from rallytime/`bp-43020`_ + + * 0eb15a1 test with gem that appears to be abandoned + + * 4150b09 Merge pull request `#43033`_ from rallytime/`bp-42760`_ + + * 3e3f7f5 Catch TypeError thrown by m2crypto when parsing missing subjects in certificate files. + + * b124d36 Merge pull request `#43032`_ from rallytime/`bp-42547`_ + + * ea4d7f4 Updated testinfra modules to work with more recent versions + + * a88386a Merge pull request `#43027`_ from pabloh007/fix-docker-save-push-2016-11 + + * d0fd949 Fixes ignore push flag for docker.push module issue `#42992`_ + + * 51d1684 Merge pull request `#42890`_ from DSRCorporation/bugs/42627_salt-cp + + * cfddbf1 Apply code review: update the doc + + * afedd3b Typos and version fixes in the doc. + + * 9fedf60 Fixed 'test_valid_docs' test. + + * 9993886 Make chunked mode in salt-cp optional (disabled by default). + + * b3c253c Merge pull request `#43009`_ from rallytime/merge-2016.11 + + * 566ba4f Merge branch '2016.3' into '2016.11' + + * 13b8637 Merge pull request `#42942`_ from Ch3LL/2016.3.6_follow_up + + * f281e17 move additional minion config options to 2016.3.8 release notes + + * 168604b remove merge conflict + + * 8a07d95 update release notes with cve number + + * 149633f Add release notes for 2016.3.7 release + + * 7a4cddc Add clean_id function to salt.utils.verify.py + + * bbb1b29 Merge pull request `#42954`_ from Ch3LL/latest_2016.3 + + * b551e66 [2016.3] Bump latest and previous versions + + * 5d5edc5 Merge pull request `#42949`_ from Ch3LL/2016.3.7_docs + + * d75d374 Add Security Notice to 2016.3.7 Release Notes + + * 37c63e7 Merge pull request `#43021`_ from terminalmage/fix-network-test + + * 4089b7b Use socket.AF_INET6 to get the correct value instead of doing an OS check + + * 8f64232 Merge pull request `#43019`_ from rallytime/bootstrap_2017.08.17 + + * 2f762b3 Update bootstrap script to latest stable: v2017.08.17 + + * ff1caeee Merge pull request `#43014`_ from Ch3LL/fix_network_mac + + * b8eee44 Change AF_INET6 family for mac in test_host_to_ips + +- **PR** `#43035`_: (*rallytime*) [2017.7] Merge forward from 2017.7.1 to 2017.7 + @ *2017-08-18T12:58:17Z* + + - **PR** `#42948`_: (*Ch3LL*) [2017.7.1] Add clean_id function to salt.utils.verify.py + | refs: `#43035`_ + - **PR** `#42945`_: (*Ch3LL*) [2017.7] Add clean_id function to salt.utils.verify.py + | refs: `#43035`_ + * d15b0ca Merge pull request `#43035`_ from rallytime/merge-2017.7 + * 756128a Merge branch '2017.7.1' into '2017.7' + + * ab1b099 Merge pull request `#42948`_ from Ch3LL/2017.7.0_follow_up + +- **PR** `#43034`_: (*rallytime*) Back-port `#43002`_ to 2017.7 + @ *2017-08-17T23:18:16Z* + + - **ISSUE** `#42989`_: (*blbradley*) GitFS GitPython performance regression in 2017.7.1 + | refs: `#43002`_ `#43002`_ + - **PR** `#43002`_: (*the-glu*) Try to fix `#42989`_ + | refs: `#43034`_ + * bcbb973 Merge pull request `#43034`_ from rallytime/`bp-43002`_ + * 350c076 Try to fix `#42989`_ by doing sslVerify and refspecs for origin remote only if there is no remotes + +- **PR** `#42958`_: (*gtmanfred*) runit module should also be loaded as runit + @ *2017-08-17T22:30:23Z* + + - **ISSUE** `#42375`_: (*dragonpaw*) salt.modules.*.__virtualname__ doens't work as documented. + | refs: `#42523`_ `#42958`_ + * 9182f55 Merge pull request `#42958`_ from gtmanfred/2017.7 + * fd68746 runit module should also be loaded as runit + +- **PR** `#43031`_: (*gtmanfred*) use a ruby gem that doesn't have dependencies + | refs: `#43048`_ + @ *2017-08-17T22:26:25Z* + + * 5985cc4 Merge pull request `#43031`_ from gtmanfred/test_gem + * ba80a7d use a ruby gem that doesn't have dependencies + +- **PR** `#43030`_: (*rallytime*) Small cleanup to dockermod.save + @ *2017-08-17T22:26:00Z* + + * 246176b Merge pull request `#43030`_ from rallytime/dockermod-minor-change + * d6a5e85 Small cleanup to dockermod.save + +- **PR** `#42993`_: (*pabloh007*) Fixes ignored push flag for docker.push module issue `#42992`_ + @ *2017-08-17T18:50:37Z* + + - **ISSUE** `#42992`_: (*pabloh007*) docker.save flag push does is ignored + * 1600011 Merge pull request `#42993`_ from pabloh007/fix-docker-save-push + * fe7554c Fixes ignored push flag for docker.push module issue `#42992`_ + +- **PR** `#42967`_: (*terminalmage*) Fix bug in on_header callback when no Content-Type is found in headers + @ *2017-08-17T18:48:52Z* + + - **ISSUE** `#42941`_: (*danlsgiga*) pkg.installed fails on installing from HTTPS rpm source + | refs: `#42967`_ + * 9009a97 Merge pull request `#42967`_ from terminalmage/issue42941 + * b838460 Fix bug in on_header callback when no Content-Type is found in headers + +- **PR** `#43016`_: (*gtmanfred*) service should return false on exception + @ *2017-08-17T18:08:05Z* + + - **ISSUE** `#43008`_: (*fillarios*) states.service.running always succeeds when watched state has changes + | refs: `#43016`_ + * 58f070d Merge pull request `#43016`_ from gtmanfred/service + * 21c264f service should return false on exception + +- **PR** `#43020`_: (*gtmanfred*) test with gem that appears to be abandoned + | refs: `#43026`_ + @ *2017-08-17T16:40:41Z* + + * 973d288 Merge pull request `#43020`_ from gtmanfred/test_gem + * 0a1f40a test with gem that appears to be abandoned + +- **PR** `#42999`_: (*garethgreenaway*) Fixes to slack engine + @ *2017-08-17T15:46:24Z* + + * 9cd0607 Merge pull request `#42999`_ from garethgreenaway/slack_engine_allow_editing_messages + * 0ece2a8 Fixing a bug that prevented editing Slack messages and having the commands resent to the Slack engine. + +- **PR** `#43010`_: (*rallytime*) [2017.7] Merge forward from 2016.11 to 2017.7 + @ *2017-08-17T15:10:29Z* + + - **ISSUE** `#42803`_: (*gmcwhistler*) master_type: str, not working as expected, parent salt-minion process dies. + | refs: `#42848`_ + - **ISSUE** `#42753`_: (*grichmond-salt*) SaltReqTimeout Error on Some Minions when One Master in a Multi-Master Configuration is Unavailable + | refs: `#42848`_ + - **ISSUE** `#42644`_: (*stamak*) nova salt-cloud -P Private IPs returned, but not public. Checking for misidentified IPs + | refs: `#42940`_ + - **ISSUE** `#38839`_: (*DaveOHenry*) Invoking runner.cloud.action via reactor sls fails + | refs: `#42291`_ + - **PR** `#42968`_: (*vutny*) [DOCS] Fix link to Salt Cloud Feature Matrix + - **PR** `#42959`_: (*rallytime*) Back-port `#42883`_ to 2016.11 + - **PR** `#42952`_: (*Ch3LL*) [2016.11] Bump latest and previous versions + - **PR** `#42950`_: (*Ch3LL*) Add Security Notice to 2016.11.7 Release Notes + - **PR** `#42944`_: (*Ch3LL*) [2016.11] Add clean_id function to salt.utils.verify.py + - **PR** `#42940`_: (*gtmanfred*) create new ip address before checking list of allocated ips + - **PR** `#42919`_: (*rallytime*) Back-port `#42871`_ to 2016.11 + - **PR** `#42918`_: (*rallytime*) Back-port `#42848`_ to 2016.11 + - **PR** `#42883`_: (*rallytime*) Fix failing boto tests + | refs: `#42959`_ + - **PR** `#42871`_: (*amalleo25*) Update joyent.rst + | refs: `#42919`_ + - **PR** `#42861`_: (*twangboy*) Fix pkg.install salt-minion using salt-call + - **PR** `#42848`_: (*DmitryKuzmenko*) Execute fire_master asynchronously in the main minion thread. + | refs: `#42918`_ + - **PR** `#42836`_: (*aneeshusa*) Backport salt.utils.versions from develop to 2016.11 + - **PR** `#42835`_: (*aneeshusa*) Fix typo in utils/versions.py module + | refs: `#42836`_ + - **PR** `#42798`_: (*s-sebastian*) Update return data before calling returners + - **PR** `#42291`_: (*vutny*) Fix `#38839`_: remove `state` from Reactor runner kwags + * 31627a9 Merge pull request `#43010`_ from rallytime/merge-2017.7 + * 8a0f948 Merge branch '2016.11' into '2017.7' + + * 1ee9499 Merge pull request `#42968`_ from vutny/doc-salt-cloud-ref + + * 44ed53b [DOCS] Fix link to Salt Cloud Feature Matrix + + * 923f974 Merge pull request `#42291`_ from vutny/`fix-38839`_ + + * 5f8f98a Fix `#38839`_: remove `state` from Reactor runner kwags + + * c20bc7d Merge pull request `#42940`_ from gtmanfred/2016.11 + + * 253e216 fix IP address spelling + + * bd63074 create new ip address before checking list of allocated ips + + * d6496ec Merge pull request `#42959`_ from rallytime/`bp-42883`_ + + * c6b9ca4 Lint fix: add missing space + + * 5597b1a Skip 2 failing tests in Python 3 due to upstream bugs + + * a0b19bd Update account id value in boto_secgroup module unit test + + * 60b406e @mock_elb needs to be changed to @mock_elb_deprecated as well + + * 6ae1111 Replace @mock_ec2 calls with @mock_ec2_deprecated calls + + * 6366e05 Merge pull request `#42944`_ from Ch3LL/2016.11.6_follow_up + + * 7e0a20a Add release notes for 2016.11.7 release + + * 63823f8 Add clean_id function to salt.utils.verify.py + + * 49d339c Merge pull request `#42952`_ from Ch3LL/latest_2016.11 + + * 74e7055 [2016.11] Bump latest and previous versions + + * b0d2e05 Merge pull request `#42950`_ from Ch3LL/2016.11.7_docs + + * a6f902d Add Security Notice to 2016.11.77 Release Notes + + * c0ff69f Merge pull request `#42836`_ from lyft/backport-utils.versions-to-2016.11 + + * 86ce700 Backport salt.utils.versions from develop to 2016.11 + + * 64a79dd Merge pull request `#42919`_ from rallytime/`bp-42871`_ + + * 4e46c96 Update joyent.rst + + * bea8ec1 Merge pull request `#42918`_ from rallytime/`bp-42848`_ + + * cdb4812 Make lint happier. + + * 62eca9b Execute fire_master asynchronously in the main minion thread. + + * 52bce32 Merge pull request `#42861`_ from twangboy/win_pkg_install_salt + + * 0d3789f Fix pkg.install salt-minion using salt-call + + * b9f4f87 Merge pull request `#42798`_ from s-sebastian/2016.11 + + * 1cc8659 Update return data before calling returners + +- **PR** `#42884`_: (*Giandom*) Convert to dict type the pillar string value passed from slack + @ *2017-08-16T22:30:43Z* + + - **ISSUE** `#42842`_: (*Giandom*) retreive kwargs passed with slack engine + | refs: `#42884`_ + * 82be9dc Merge pull request `#42884`_ from Giandom/2017.7.1-fix-slack-engine-pillar-args + * 80fd733 Update slack.py + +- **PR** `#42963`_: (*twangboy*) Fix `unit.test_fileclient` for Windows + @ *2017-08-16T14:18:18Z* + + * 42bd553 Merge pull request `#42963`_ from twangboy/win_unit_test_fileclient + * e9febe4 Fix unit.test_fileclient + +- **PR** `#42964`_: (*twangboy*) Fix `salt.utils.recursive_copy` for Windows + @ *2017-08-16T14:17:27Z* + + * 7dddeee Merge pull request `#42964`_ from twangboy/win_fix_recursive_copy + * 121cd4e Fix `salt.utils.recursive_copy` for Windows + +- **PR** `#42946`_: (*mirceaulinic*) extension_modules should default to $CACHE_DIR/proxy/extmods + @ *2017-08-15T21:26:36Z* + + - **ISSUE** `#42943`_: (*mirceaulinic*) `extension_modules` defaulting to `/var/cache/minion` although running under proxy minion + | refs: `#42946`_ + * 6da4d1d Merge pull request `#42946`_ from cloudflare/px_extmods_42943 + * 73f9135 extension_modules should default to /proxy/extmods + +- **PR** `#42945`_: (*Ch3LL*) [2017.7] Add clean_id function to salt.utils.verify.py + | refs: `#43035`_ + @ *2017-08-15T18:04:20Z* + + * 95645d4 Merge pull request `#42945`_ from Ch3LL/2017.7.0_follow_up + * dcd9204 remove extra doc + + * 693a504 update release notes with cve number + +- **PR** `#42812`_: (*terminalmage*) Update custom YAML loader tests to properly test unicode literals + @ *2017-08-15T17:50:22Z* + + - **ISSUE** `#42427`_: (*grichmond-salt*) Issue Passing Variables created from load_json as Inline Pillar Between States + | refs: `#42435`_ + - **PR** `#42435`_: (*terminalmage*) Modify our custom YAML loader to treat unicode literals as unicode strings + | refs: `#42812`_ + * 47ff9d5 Merge pull request `#42812`_ from terminalmage/yaml-loader-tests + * 9d8486a Add test for custom YAML loader with unicode literal strings + + * a0118bc Remove bytestrings and use textwrap.dedent for readability + +- **PR** `#42953`_: (*Ch3LL*) [2017.7] Bump latest and previous versions + @ *2017-08-15T17:23:28Z* + + * 5d0c219 Merge pull request `#42953`_ from Ch3LL/latest_2017.7 + * cbecf65 [2017.7] Bump latest and previous versions + +- **PR** `#42951`_: (*Ch3LL*) Add Security Notice to 2017.7.1 Release Notes + @ *2017-08-15T16:49:56Z* + + * 730e71d Merge pull request `#42951`_ from Ch3LL/2017.7.1_docs + * 1d8f827 Add Security Notice to 2017.7.1 Release Notes + +- **PR** `#42868`_: (*carsonoid*) Stub out required functions in redis_cache + @ *2017-08-15T14:33:54Z* + + * c1c8cb9 Merge pull request `#42868`_ from carsonoid/redisjobcachefix + * 885bee2 Stub out required functions for redis cache + +- **PR** `#42810`_: (*amendlik*) Ignore error values when listing Windows SNMP community strings + @ *2017-08-15T03:55:15Z* + + * e192d6e Merge pull request `#42810`_ from amendlik/win-snmp-community + * dc20e46 Ignore error values when listing Windows SNMP community strings + +- **PR** `#42920`_: (*cachedout*) pid_race + @ *2017-08-15T03:49:10Z* + + * a1817f1 Merge pull request `#42920`_ from cachedout/pid_race + * 5e930b8 If we catch the pid file in a transistory state, return None + +- **PR** `#42925`_: (*terminalmage*) Add debug logging to troubleshoot test failures + @ *2017-08-15T03:47:51Z* + + * 11a33fe Merge pull request `#42925`_ from terminalmage/f26-debug-logging + * 8165f46 Add debug logging to troubleshoot test failures + +- **PR** `#42913`_: (*twangboy*) Change service shutdown timeouts for salt-minion service (Windows) + @ *2017-08-14T20:55:24Z* + + * a537197 Merge pull request `#42913`_ from twangboy/win_change_timeout + * ffb23fb Remove the line that wipes out the cache + + * a3becf8 Change service shutdown timeouts + +- **PR** `#42800`_: (*skizunov*) Fix exception when master_type=disable + @ *2017-08-14T20:53:38Z* + + * ca0555f Merge pull request `#42800`_ from skizunov/develop6 + * fa58220 Fix exception when master_type=disable + +- **PR** `#42679`_: (*mirceaulinic*) Add multiprocessing option for NAPALM proxy + @ *2017-08-14T20:45:06Z* + + * 3af264b Merge pull request `#42679`_ from cloudflare/napalm-multiprocessing + * 9c4566d multiprocessing option tagged for 2017.7.2 + + * 37bca1b Add multiprocessing option for NAPALM proxy + + * a2565ba Add new napalm option: multiprocessing + +- **PR** `#42657`_: (*nhavens*) back-port `#42612`_ to 2017.7 + @ *2017-08-14T19:42:26Z* + + - **ISSUE** `#42611`_: (*nhavens*) selinux.boolean state does not return changes + | refs: `#42612`_ + - **PR** `#42612`_: (*nhavens*) fix for issue `#42611`_ + | refs: `#42657`_ + * 4fcdab3 Merge pull request `#42657`_ from nhavens/2017.7 + * d73c4b5 back-port `#42612`_ to 2017.7 + +- **PR** `#42709`_: (*whiteinge*) Add token_expire_user_override link to auth runner docstring + @ *2017-08-14T19:03:06Z* + + * d2b6ce3 Merge pull request `#42709`_ from whiteinge/doc-token_expire_user_override + * c7ea631 Add more docs on the token_expire param + + * 4a9f6ba Add token_expire_user_override link to auth runner docstring + +- **PR** `#42848`_: (*DmitryKuzmenko*) Execute fire_master asynchronously in the main minion thread. + | refs: `#42918`_ + @ *2017-08-14T18:28:38Z* + + - **ISSUE** `#42803`_: (*gmcwhistler*) master_type: str, not working as expected, parent salt-minion process dies. + | refs: `#42848`_ + - **ISSUE** `#42753`_: (*grichmond-salt*) SaltReqTimeout Error on Some Minions when One Master in a Multi-Master Configuration is Unavailable + | refs: `#42848`_ + * c6a7bf0 Merge pull request `#42848`_ from DSRCorporation/bugs/42753_mmaster_timeout + * 7f5412c Make lint happier. + + * ff66b7a Execute fire_master asynchronously in the main minion thread. + +- **PR** `#42911`_: (*gtmanfred*) cloud driver isn't a provider + @ *2017-08-14T17:47:16Z* + + * 6a3279e Merge pull request `#42911`_ from gtmanfred/2017.7 + * 99046b4 cloud driver isn't a provider + +- **PR** `#42860`_: (*skizunov*) hash_and_stat_file should return a 2-tuple + @ *2017-08-14T15:44:54Z* + + * 4456f73 Merge pull request `#42860`_ from skizunov/develop7 + * 5f85a03 hash_and_stat_file should return a 2-tuple + +- **PR** `#42889`_: (*rallytime*) [2017.7] Merge forward from 2016.11 to 2017.7 + @ *2017-08-14T14:16:20Z* + + - **ISSUE** `#41976`_: (*abulford*) dockerng network states do not respect test=True + | refs: `#41977`_ `#41977`_ + - **ISSUE** `#41770`_: (*Ch3LL*) NPM v5 incompatible with salt.modules.cache_list + | refs: `#42856`_ + - **ISSUE** `#475`_: (*thatch45*) Change yaml to use C bindings + | refs: `#42856`_ + - **PR** `#42886`_: (*sarcasticadmin*) Adding missing output flags to salt cli docs + - **PR** `#42882`_: (*gtmanfred*) make sure cmd is not run when npm isn't installed + - **PR** `#42877`_: (*terminalmage*) Add virtual func for cron state module + - **PR** `#42864`_: (*whiteinge*) Make syndic_log_file respect root_dir setting + - **PR** `#42859`_: (*terminalmage*) Add note about git CLI requirement for GitPython to GitFS tutorial + - **PR** `#42856`_: (*gtmanfred*) skip cache_clean test if npm version is >= 5.0.0 + - **PR** `#42788`_: (*amendlik*) Remove waits and retries from Saltify deployment + - **PR** `#41977`_: (*abulford*) Fix dockerng.network_* ignoring of tests=True + * c6ca7d6 Merge pull request `#42889`_ from rallytime/merge-2017.7 + * fb7117f Use salt.utils.versions.LooseVersion instead of distutils + + * 29ff19c Merge branch '2016.11' into '2017.7' + + * c15d003 Merge pull request `#41977`_ from redmatter/fix-dockerng-network-ignores-test + + * 1cc2aa5 Fix dockerng.network_* ignoring of tests=True + + * 3b9c3c5 Merge pull request `#42886`_ from sarcasticadmin/adding_docs_salt_outputs + + * 744bf95 Adding missing output flags to salt cli + + * e5b98c8 Merge pull request `#42882`_ from gtmanfred/2016.11 + + * da3402a make sure cmd is not run when npm isn't installed + + * 5962c95 Merge pull request `#42788`_ from amendlik/saltify-timeout + + * 928b523 Remove waits and retries from Saltify deployment + + * 227ecdd Merge pull request `#42877`_ from terminalmage/add-cron-state-virtual + + * f1de196 Add virtual func for cron state module + + * ab9f6ce Merge pull request `#42859`_ from terminalmage/gitpython-git-cli-note + + * 35e05c9 Add note about git CLI requirement for GitPython to GitFS tutorial + + * 682b4a8 Merge pull request `#42856`_ from gtmanfred/2016.11 + + * b458b89 skip cache_clean test if npm version is >= 5.0.0 + + * 01ea854 Merge pull request `#42864`_ from whiteinge/syndic-log-root_dir + + * 4b1f55d Make syndic_log_file respect root_dir setting + +- **PR** `#42898`_: (*mirceaulinic*) Minor eos doc correction + @ *2017-08-14T13:42:21Z* + + * 4b6fe2e Merge pull request `#42898`_ from mirceaulinic/patch-11 + * 93be79a Index eos under the installation instructions list + + * f903e7b Minor eos doc correction + +- **PR** `#42883`_: (*rallytime*) Fix failing boto tests + | refs: `#42959`_ + @ *2017-08-11T20:29:12Z* + + * 1764878 Merge pull request `#42883`_ from rallytime/fix-boto-tests + * 6a7bf99 Lint fix: add missing space + + * 4364322 Skip 2 failing tests in Python 3 due to upstream bugs + + * 7f46603 Update account id value in boto_secgroup module unit test + + * 7c1d493 @mock_elb needs to be changed to @mock_elb_deprecated as well + + * 3055e17 Replace @mock_ec2 calls with @mock_ec2_deprecated calls + +- **PR** `#42885`_: (*terminalmage*) Move weird tearDown test to an actual tearDown + @ *2017-08-11T19:14:42Z* + + * b21778e Merge pull request `#42885`_ from terminalmage/fix-f26-tests + * 462d653 Move weird tearDown test to an actual tearDown + +- **PR** `#42887`_: (*rallytime*) Remove extraneous "deprecated" notation + @ *2017-08-11T18:34:25Z* + + - **ISSUE** `#42870`_: (*boltronics*) webutil.useradd marked as deprecated:: 2016.3.0 by mistake? + | refs: `#42887`_ + * 9868ab6 Merge pull request `#42887`_ from rallytime/`fix-42870`_ + * 71e7581 Remove extraneous "deprecated" notation + +- **PR** `#42881`_: (*gtmanfred*) fix vmware for python 3.4.2 in salt.utils.vmware + @ *2017-08-11T17:52:29Z* + + * da71f2a Merge pull request `#42881`_ from gtmanfred/vmware + * 05ecc6a fix vmware for python 3.4.2 in salt.utils.vmware + +- **PR** `#42845`_: (*brejoc*) API changes for Kubernetes version 2.0.0 + | refs: `#43039`_ + @ *2017-08-11T14:04:30Z* + + - **ISSUE** `#42843`_: (*brejoc*) Kubernetes module won't work with Kubernetes Python client > 1.0.2 + | refs: `#42845`_ + * c7750d5 Merge pull request `#42845`_ from brejoc/updates-for-kubernetes-2.0.0 + * 81674aa Version info in :optdepends: not needed anymore + + * 7199550 Not depending on specific K8s version anymore + + * d8f7d7a API changes for Kubernetes version 2.0.0 + +- **PR** `#42678`_: (*frankiexyz*) Add eos.rst in the installation guide + @ *2017-08-11T13:58:37Z* + + * 459fded Merge pull request `#42678`_ from frankiexyz/2017.7 + * 1598571 Add eos.rst in the installation guide + +- **PR** `#42778`_: (*gtmanfred*) make sure to use the correct out_file + @ *2017-08-11T13:44:48Z* + + - **ISSUE** `#42646`_: (*gmacon*) SPM fails to install multiple packages + | refs: `#42778`_ + * 4ce96eb Merge pull request `#42778`_ from gtmanfred/spm + * 7ef691e make sure to use the correct out_file + +- **PR** `#42857`_: (*gtmanfred*) use older name if _create_unverified_context is unvailable + @ *2017-08-11T13:37:59Z* + + - **ISSUE** `#480`_: (*zyluo*) PEP8 types clean-up + | refs: `#42857`_ + * 3d05d89 Merge pull request `#42857`_ from gtmanfred/vmware + * c1f673e use older name if _create_unverified_context is unvailable + +- **PR** `#42866`_: (*twangboy*) Change to GitPython version 2.1.1 + @ *2017-08-11T13:23:52Z* + + * 7e8cfff Merge pull request `#42866`_ from twangboy/osx_downgrade_gitpython + * 28053a8 Change GitPython version to 2.1.1 + +- **PR** `#42855`_: (*rallytime*) [2017.7] Merge forward from 2016.11 to 2017.7 + @ *2017-08-10T21:40:39Z* + + - **ISSUE** `#42747`_: (*whiteinge*) Outputters mutate data which can be a problem for Runners and perhaps other things + | refs: `#42748`_ + - **ISSUE** `#42731`_: (*infoveinx*) http.query template_data render exception + | refs: `#42804`_ + - **ISSUE** `#42690`_: (*ChristianBeer*) git.latest state with remote set fails on first try + | refs: `#42694`_ + - **ISSUE** `#42683`_: (*rgcosma*) Gluster module broken in 2017.7 + | refs: `#42806`_ + - **ISSUE** `#42600`_: (*twangboy*) Unable to set 'Not Configured' using win_lgpo execution module + | refs: `#42744`_ `#42794`_ `#42795`_ + - **PR** `#42851`_: (*terminalmage*) Backport `#42651`_ to 2016.11 + - **PR** `#42838`_: (*twangboy*) Document requirements for win_pki + - **PR** `#42829`_: (*twangboy*) Fix passing version in pkgs as shown in docs + - **PR** `#42826`_: (*terminalmage*) Fix misspelling of "versions" + - **PR** `#42806`_: (*rallytime*) Update doc references in glusterfs.volume_present + - **PR** `#42805`_: (*rallytime*) Back-port `#42552`_ to 2016.11 + - **PR** `#42804`_: (*rallytime*) Back-port `#42784`_ to 2016.11 + - **PR** `#42795`_: (*lomeroe*) backport `#42744`_ to 2016.11 + - **PR** `#42786`_: (*Ch3LL*) Fix typo for template_dict in http docs + - **PR** `#42784`_: (*gtmanfred*) only read file if ret is not a string in http.query + | refs: `#42804`_ + - **PR** `#42764`_: (*amendlik*) Fix infinite loop with salt-cloud and Windows nodes + - **PR** `#42748`_: (*whiteinge*) Workaround Orchestrate problem that highstate outputter mutates data + - **PR** `#42744`_: (*lomeroe*) fix `#42600`_ in develop + | refs: `#42794`_ `#42795`_ + - **PR** `#42694`_: (*gtmanfred*) allow adding extra remotes to a repository + - **PR** `#42651`_: (*gtmanfred*) python2- prefix for fedora 26 packages + - **PR** `#42552`_: (*remijouannet*) update consul module following this documentation https://www.consul.… + | refs: `#42805`_ + * 3ce1863 Merge pull request `#42855`_ from rallytime/merge-2017.7 + * 08bbcf5 Merge branch '2016.11' into '2017.7' + + * 2dde1f7 Merge pull request `#42851`_ from terminalmage/`bp-42651`_ + + * a3da86e fix syntax + + * 6ecdbce make sure names are correct + + * f83b553 add py3 for versionlock + + * 21934f6 python2- prefix for fedora 26 packages + + * c746f79 Merge pull request `#42806`_ from rallytime/`fix-42683`_ + + * 8c8640d Update doc references in glusterfs.volume_present + + * 27a8a26 Merge pull request `#42829`_ from twangboy/win_pkg_fix_install + + * 83b9b23 Add winrepo to docs about supporting versions in pkgs + + * 81fefa6 Add ability to pass version in pkgs list + + * 3c3ac6a Merge pull request `#42838`_ from twangboy/win_doc_pki + + * f0a1d06 Standardize PKI Client + + * 7de687a Document requirements for win_pki + + * b3e2ae3 Merge pull request `#42805`_ from rallytime/`bp-42552`_ + + * 5a91c1f update consul module following this documentation https://www.consul.io/api/acl.html + + * d2ee793 Merge pull request `#42804`_ from rallytime/`bp-42784`_ + + * dbd29e4 only read file if it is not a string + + * 4cbf805 Merge pull request `#42826`_ from terminalmage/fix-spelling + + * 00f9314 Fix misspelling of "versions" + + * de997ed Merge pull request `#42786`_ from Ch3LL/fix_typo + + * 90a2fb6 Fix typo for template_dict in http docs + + * bf6153e Merge pull request `#42795`_ from lomeroe/`bp-42744`__201611 + + * 695f8c1 fix `#42600`_ in develop + + * 61fad97 Merge pull request `#42748`_ from whiteinge/save-before-output + + * de60b77 Workaround Orchestrate problem that highstate outputter mutates data + + * a4e3e7e Merge pull request `#42764`_ from amendlik/cloud-win-loop + + * f3dcfca Fix infinite loops on failed Windows deployments + + * da85326 Merge pull request `#42694`_ from gtmanfred/2016.11 + + * 1a0457a allow adding extra remotes to a repository + +- **PR** `#42808`_: (*terminalmage*) Fix regression in yum/dnf version specification + @ *2017-08-10T15:59:22Z* + + - **ISSUE** `#42774`_: (*rossengeorgiev*) pkg.installed succeeds, but fails when you specify package version + | refs: `#42808`_ + * f954f4f Merge pull request `#42808`_ from terminalmage/issue42774 + * c69f17d Add integration test for `#42774`_ + + * 78d826d Fix regression in yum/dnf version specification + +- **PR** `#42807`_: (*rallytime*) Update modules --> states in kubernetes doc module + @ *2017-08-10T14:10:40Z* + + - **ISSUE** `#42639`_: (*amnonbc*) k8s module needs a way to manage configmaps + | refs: `#42807`_ + * d9b0f44 Merge pull request `#42807`_ from rallytime/`fix-42639`_ + * 152eb88 Update modules --> states in kubernetes doc module + +- **PR** `#42841`_: (*Mapel88*) Fix bug `#42818`_ in win_iis module + @ *2017-08-10T13:44:21Z* + + - **ISSUE** `#42818`_: (*Mapel88*) Bug in win_iis module - "create_cert_binding" + | refs: `#42841`_ + * b8c7bda Merge pull request `#42841`_ from Mapel88/patch-1 + * 497241f Fix bug `#42818`_ in win_iis module + +- **PR** `#42782`_: (*rallytime*) Add a cmp compatibility function utility + @ *2017-08-09T22:37:29Z* + + - **ISSUE** `#42697`_: (*Ch3LL*) [Python3] NameError when running salt-run manage.versions + | refs: `#42782`_ + * 135f952 Merge pull request `#42782`_ from rallytime/`fix-42697`_ + * d707f94 Update all other calls to "cmp" function + + * 5605104 Add a cmp compatibility function utility + +- **PR** `#42784`_: (*gtmanfred*) only read file if ret is not a string in http.query + | refs: `#42804`_ + @ *2017-08-08T17:20:13Z* + + * ac75222 Merge pull request `#42784`_ from gtmanfred/http + * d397c90 only read file if it is not a string + +- **PR** `#42794`_: (*lomeroe*) Backport `#42744`_ to 2017.7 + @ *2017-08-08T17:16:31Z* + + - **ISSUE** `#42600`_: (*twangboy*) Unable to set 'Not Configured' using win_lgpo execution module + | refs: `#42744`_ `#42794`_ `#42795`_ + - **PR** `#42744`_: (*lomeroe*) fix `#42600`_ in develop + | refs: `#42794`_ `#42795`_ + * 44995b1 Merge pull request `#42794`_ from lomeroe/`bp-42744`_ + * 0acffc6 fix `#42600`_ in develop + +- **PR** `#42708`_: (*cro*) Do not change the arguments of the function when memoizing + @ *2017-08-08T13:47:01Z* + + - **ISSUE** `#42707`_: (*cro*) Service module and state fails on FreeBSD + | refs: `#42708`_ + * dcf474c Merge pull request `#42708`_ from cro/dont_change_args_during_memoize + * a260e91 Do not change the arguments of the function when memoizing + +- **PR** `#42783`_: (*rallytime*) Sort lists before comparing them in python 3 unit test + @ *2017-08-08T13:25:15Z* + + - **PR** `#42206`_: (*rallytime*) [PY3] Fix test that is flaky in Python 3 + | refs: `#42783`_ + * ddb671b Merge pull request `#42783`_ from rallytime/fix-flaky-py3-test + * 998834f Sort lists before compairing them in python 3 unit test + +- **PR** `#42721`_: (*hibbert*) Allow no ip sg + @ *2017-08-07T22:07:18Z* + + * d69822f Merge pull request `#42721`_ from hibbert/allow_no_ip_sg + * f582568 allow_no_ip_sg: Allow user to not supply ipaddress or securitygroups when running boto_efs.create_mount_target + +- **PR** `#42769`_: (*terminalmage*) Fix domainname parameter input translation + @ *2017-08-07T20:46:07Z* + + - **ISSUE** `#42538`_: (*marnovdm*) docker_container.running issue since 2017.7.0: passing domainname gives Error 500: json: cannot unmarshal array into Go value of type string + | refs: `#42769`_ + * bf7938f Merge pull request `#42769`_ from terminalmage/issue42538 + * 665de2d Fix domainname parameter input translation + +- **PR** `#42388`_: (*The-Loeki*) pillar.items pillar_env & pillar_override are never used + @ *2017-08-07T17:51:48Z* + + * 7bf2cdb Merge pull request `#42388`_ from The-Loeki/patch-1 + * 664f4b5 pillar.items pillar_env & pillar_override are never used + +- **PR** `#42770`_: (*rallytime*) [2017.7] Merge forward from 2017.7.1 to 2017.7 + @ *2017-08-07T16:21:45Z* + + * 9a8c9eb Merge pull request `#42770`_ from rallytime/merge-2017.7.1-into-2017.7 + * 6d17c9d Merge branch '2017.7.1' into '2017.7' + +- **PR** `#42768`_: (*rallytime*) [2017.7] Merge forward from 2016.11 to 2017.7 + @ *2017-08-07T16:21:17Z* + + - **ISSUE** `#42686`_: (*gilbsgilbs*) Unable to set multiple RabbitMQ tags + | refs: `#42693`_ `#42693`_ + - **ISSUE** `#42642`_: (*githubcdr*) state.augeas + | refs: `#42669`_ `#43202`_ + - **ISSUE** `#41433`_: (*sbojarski*) boto_cfn.present fails when reporting error for failed state + | refs: `#42574`_ + - **PR** `#42693`_: (*gilbsgilbs*) Fix RabbitMQ tags not properly set. + - **PR** `#42669`_: (*garethgreenaway*) [2016.11] Fixes to augeas module + - **PR** `#42655`_: (*whiteinge*) Reenable cpstats for rest_cherrypy + - **PR** `#42629`_: (*xiaoanyunfei*) tornado api + - **PR** `#42623`_: (*terminalmage*) Fix unicode constructor in custom YAML loader + - **PR** `#42574`_: (*sbojarski*) Fixed error reporting in "boto_cfn.present" function. + - **PR** `#33806`_: (*cachedout*) Work around upstream cherrypy bug + | refs: `#42655`_ + * c765e52 Merge pull request `#42768`_ from rallytime/merge-2017.7 + * 0f75482 Merge branch '2016.11' into '2017.7' + + * 7b2119f Merge pull request `#42669`_ from garethgreenaway/42642_2016_11_augeas_module_fix + + * 2441308 Updating the call to shlex_split to pass the posix=False argument so that quotes are preserved. + + * 3072576 Merge pull request `#42629`_ from xiaoanyunfei/tornadoapi + + * 1e13383 tornado api + + * f0f00fc Merge pull request `#42655`_ from whiteinge/rest_cherrypy-reenable-stats + + * deb6316 Fix lint errors + + * 6bd91c8 Reenable cpstats for rest_cherrypy + + * 21cf15f Merge pull request `#42693`_ from gilbsgilbs/fix-rabbitmq-tags + + * 78fccdc Cast to list in case tags is a tuple. + + * 287b57b Fix RabbitMQ tags not properly set. + + * f2b0c9b Merge pull request `#42574`_ from sbojarski/boto-cfn-error-reporting + + * 5c945f1 Fix debug message in "boto_cfn._validate" function. + + * 181a1be Fixed error reporting in "boto_cfn.present" function. + + * bc1effc Merge pull request `#42623`_ from terminalmage/fix-unicode-constructor + + * fcf4588 Fix unicode constructor in custom YAML loader + +- **PR** `#42651`_: (*gtmanfred*) python2- prefix for fedora 26 packages + @ *2017-08-07T14:35:04Z* + + * 3f5827f Merge pull request `#42651`_ from gtmanfred/2017.7 + * 8784899 fix syntax + + * 178cc1b make sure names are correct + + * f179b97 add py3 for versionlock + + * 1958d18 python2- prefix for fedora 26 packages + +- **PR** `#42689`_: (*hibbert*) boto_efs_fix_tags: Fix `#42688`_ invalid type for parameter tags + @ *2017-08-06T17:47:07Z* + + - **ISSUE** `#42688`_: (*hibbert*) salt.modules.boto_efs module Invalid type for parameter Tags - type: , valid types: , + | refs: `#42689`_ + * 791248e Merge pull request `#42689`_ from hibbert/boto_efs_fix_tags + * 157fb28 boto_efs_fix_tags: Fix `#42688`_ invalid type for parameter tags + +- **PR** `#42745`_: (*terminalmage*) docker.compare_container: treat null oom_kill_disable as False + @ *2017-08-05T15:28:20Z* + + - **ISSUE** `#42705`_: (*hbruch*) salt.states.docker_container.running replaces container on subsequent runs if oom_kill_disable unsupported + | refs: `#42745`_ + * 1b34076 Merge pull request `#42745`_ from terminalmage/issue42705 + * 710bdf6 docker.compare_container: treat null oom_kill_disable as False + +- **PR** `#42704`_: (*whiteinge*) Add import to work around likely multiprocessing scoping bug + @ *2017-08-04T23:03:13Z* + + - **ISSUE** `#42649`_: (*tehsu*) local_batch no longer working in 2017.7.0, 500 error + | refs: `#42704`_ + * 5d5b220 Merge pull request `#42704`_ from whiteinge/expr_form-warn-scope-bug + * 03b675a Add import to work around likely multiprocessing scoping bug + +- **PR** `#42743`_: (*kkoppel*) Fix docker.compare_container for containers with links + @ *2017-08-04T16:00:33Z* + + - **ISSUE** `#42741`_: (*kkoppel*) docker_container.running keeps re-creating containers with links to other containers + | refs: `#42743`_ + * 888e954 Merge pull request `#42743`_ from kkoppel/fix-issue-42741 + * de6d3cc Update dockermod.py + + * 58b997c Added a helper function that removes container names from container HostConfig:Links values to enable compare_container() to make the correct decision about differences in links. + +- **PR** `#42710`_: (*gtmanfred*) use subtraction instead of or + @ *2017-08-04T15:14:14Z* + + - **ISSUE** `#42668`_: (*UtahDave*) Minions under syndics don't respond to MoM + | refs: `#42710`_ + - **ISSUE** `#42545`_: (*paul-mulvihill*) Salt-api failing to return results for minions connected via syndics. + | refs: `#42710`_ + * 03a7f9b Merge pull request `#42710`_ from gtmanfred/syndic + * 683561a use subtraction instead of or + +- **PR** `#42670`_: (*gtmanfred*) render kubernetes docs + @ *2017-08-03T20:30:56Z* + + * 005182b Merge pull request `#42670`_ from gtmanfred/kube + * bca1790 add version added info + + * 4bbfc75 render kubernetes docs + +- **PR** `#42712`_: (*twangboy*) Remove master config file from minion-only installer + @ *2017-08-03T20:25:02Z* + + * df354dd Merge pull request `#42712`_ from twangboy/win_build_pkg + * 8604312 Remove master conf in minion install + +- **PR** `#42714`_: (*cachedout*) Set fact gathering style to 'old' for test_junos + @ *2017-08-03T13:39:40Z* + + * bb1dfd4 Merge pull request `#42714`_ from cachedout/workaround_jnpr_test_bug + * 834d6c6 Set fact gathering style to 'old' for test_junos + +- **PR** `#42481`_: (*twangboy*) Fix `unit.test_crypt` for Windows + @ *2017-08-01T18:10:50Z* + + * 4c1d931 Merge pull request `#42481`_ from twangboy/win_unit_test_crypt + * 1025090 Remove chown mock, fix path seps + +- **PR** `#42654`_: (*morganwillcock*) Disable ZFS in the core grain for NetBSD + @ *2017-08-01T17:52:36Z* + + * 8bcefb5 Merge pull request `#42654`_ from morganwillcock/zfsgrain + * 49023de Disable ZFS grain on NetBSD + +- **PR** `#42453`_: (*gtmanfred*) don't pass user to makedirs on windows + @ *2017-07-31T19:57:57Z* + + - **ISSUE** `#42421`_: (*bartuss7*) archive.extracted on Windows failed when dir not exist + | refs: `#42453`_ + * 5baf265 Merge pull request `#42453`_ from gtmanfred/makedirs + * 559d432 fix tests + + * afa7a13 use logic from file.directory for makedirs + +- **PR** `#42603`_: (*twangboy*) Add runas_passwd as a global for states + @ *2017-07-31T19:49:49Z* + + * fb81e78 Merge pull request `#42603`_ from twangboy/win_fix_runas + * 0c9e400 Remove deprecation, add logic to state.py + + * 464ec34 Fix another instance of runas_passwd + + * 18d6ce4 Add global vars to cmd.call + + * 6c71ab6 Remove runas and runas_password after state run + + * 4ea264e Change to runas_password in docs + + * 61aba35 Deprecate password, make runas_password a named arg + + * 41f0f75 Add new var to list, change to runas_password + + * b9c91eb Add runas_passwd as a global for states + +- **PR** `#42541`_: (*Mareo*) Avoid confusing warning when using file.line + @ *2017-07-31T19:41:58Z* + + * 75ba23c Merge pull request `#42541`_ from epita/fix-file-line-warning + * 2fd172e Avoid confusing warning when using file.line + +- **PR** `#42625`_: (*twangboy*) Fix the list function in the win_wua execution module + @ *2017-07-31T19:27:16Z* + + * 3d328eb Merge pull request `#42625`_ from twangboy/fix_win_wua + * 1340c15 Add general usage instructions + + * 19f34bd Fix docs, formatting + + * b17495c Fix problem with list when install=True + +- **PR** `#42602`_: (*garethgreenaway*) Use superseded and deprecated configuration from pillar + @ *2017-07-31T18:53:06Z* + + - **ISSUE** `#42514`_: (*rickh563*) `module.run` does not work as expected in 2017.7.0 + | refs: `#42602`_ + * 25094ad Merge pull request `#42602`_ from garethgreenaway/42514_2017_7_superseded_deprecated_from_pillar + * 2e132da Slight update to formatting + + * 74bae13 Small update to something I missed in the first commit. Updating tests to also test for pillar values. + + * 928a480 Updating the superseded and deprecated decorators to work when specified as pillar values. + +- **PR** `#42621`_: (*rallytime*) [2017.7] Merge forward from 2016.11 to 2017.7 + @ *2017-07-28T19:45:51Z* + + - **ISSUE** `#42456`_: (*gdubroeucq*) Use yum lib + | refs: `#42586`_ + - **ISSUE** `#41982`_: (*abulford*) dockerng.network_* matches too easily + | refs: `#41988`_ `#41988`_ `#42006`_ `#42006`_ + - **PR** `#42586`_: (*gdubroeucq*) [Fix] yumpkg.py: add option to the command "check-update" + - **PR** `#42515`_: (*gtmanfred*) Allow not interpreting backslashes in the repl + - **PR** `#41988`_: (*abulford*) Fix dockerng.network_* name matching + | refs: `#42006`_ + * b7cd30d Merge pull request `#42621`_ from rallytime/merge-2017.7 + * 58dcb58 Merge branch '2016.11' into '2017.7' + + * cbf752c Merge pull request `#42515`_ from gtmanfred/backslash + + * cc4e456 Allow not interpreting backslashes in the repl + + * 5494958 Merge pull request `#42586`_ from gdubroeucq/2016.11 + + * 9c0b5cc Remove extra newline + + * d2ef448 yumpkg.py: clean + + * a96f7c0 yumpkg.py: add option to the command "check-update" + + * 6b45deb Merge pull request `#41988`_ from redmatter/fix-dockerng-network-matching + + * 9eea796 Add regression tests for `#41982`_ + + * 3369f00 Fix broken unit test test_network_absent + + * 0ef6cf6 Add trace logging of dockerng.networks result + + * 515c612 Fix dockerng.network_* name matching + +- **PR** `#42618`_: (*rallytime*) Back-port `#41690`_ to 2017.7 + @ *2017-07-28T19:27:11Z* + + - **ISSUE** `#34245`_: (*Talkless*) ini.options_present always report state change + | refs: `#41690`_ + - **PR** `#41690`_: (*m03*) Fix issue `#34245`_ with ini.options_present reporting changes + | refs: `#42618`_ + * d48749b Merge pull request `#42618`_ from rallytime/`bp-41690`_ + * 22c6a7c Improve output precision + + * ee4ea6b Fix `#34245`_ ini.options_present reporting changes + +- **PR** `#42619`_: (*rallytime*) Back-port `#42589`_ to 2017.7 + @ *2017-07-28T19:26:36Z* + + - **ISSUE** `#42588`_: (*ixs*) salt-ssh fails when using scan roster and detected minions are uncached + | refs: `#42589`_ + - **PR** `#42589`_: (*ixs*) Fix ssh-salt calls with scan roster for uncached clients + | refs: `#42619`_ + * e671242 Merge pull request `#42619`_ from rallytime/`bp-42589`_ + * cd5eb93 Fix ssh-salt calls with scan roster for uncached clients + +- **PR** `#42006`_: (*abulford*) Fix dockerng.network_* name matching + @ *2017-07-28T15:52:52Z* + + - **ISSUE** `#41982`_: (*abulford*) dockerng.network_* matches too easily + | refs: `#41988`_ `#41988`_ `#42006`_ `#42006`_ + - **PR** `#41988`_: (*abulford*) Fix dockerng.network_* name matching + | refs: `#42006`_ + * 7d385f8 Merge pull request `#42006`_ from redmatter/fix-dockerng-network-matching-2017.7 + * f83960c Lint: Remove extra line at end of file. + + * c7d364e Add regression tests for `#41982`_ + + * d31f291 Fix broken unit test test_network_absent + + * d42f781 Add trace logging of docker.networks result + + * 8c00c63 Fix dockerng.network_* name matching + +- **PR** `#42616`_: (*amendlik*) Sync cloud modules + @ *2017-07-28T15:40:36Z* + + - **ISSUE** `#12587`_: (*Katafalkas*) salt-cloud custom functions/actions + | refs: `#42616`_ + * ee8aee1 Merge pull request `#42616`_ from amendlik/sync-clouds + * ab21bd9 Sync cloud modules when saltutil.sync_all is run + +- **PR** `#42601`_: (*rallytime*) [2017.7] Merge forward from 2016.11 to 2017.7 + @ *2017-07-27T22:32:07Z* + + - **ISSUE** `#1036125`_: (**) + - **ISSUE** `#42477`_: (*aikar*) Invalid ssh_interface value prevents salt-cloud provisioning without reason of why + | refs: `#42479`_ + - **ISSUE** `#42405`_: (*felrivero*) The documentation is incorrectly compiled (PILLAR section) + | refs: `#42516`_ + - **ISSUE** `#42403`_: (*astronouth7303*) [2017.7] Pillar empty when state is applied from orchestrate + | refs: `#42433`_ + - **ISSUE** `#42375`_: (*dragonpaw*) salt.modules.*.__virtualname__ doens't work as documented. + | refs: `#42523`_ `#42958`_ + - **ISSUE** `#42371`_: (*tsaridas*) Minion unresponsive after trying to failover + | refs: `#42387`_ + - **ISSUE** `#41955`_: (*root360-AndreasUlm*) rabbitmq 3.6.10 changed output => rabbitmq-module broken + | refs: `#41968`_ + - **ISSUE** `#23516`_: (*dkiser*) BUG: cron job scheduler sporadically works + | refs: `#42077`_ + - **PR** `#42573`_: (*rallytime*) Back-port `#42433`_ to 2016.11 + - **PR** `#42571`_: (*twangboy*) Avoid loading system PYTHON* environment vars + - **PR** `#42551`_: (*binocvlar*) Remove '-s' (--script) argument to parted within align_check function + - **PR** `#42527`_: (*twangboy*) Document changes to Windows Update in Windows 10/Server 2016 + - **PR** `#42523`_: (*rallytime*) Add a mention of the True/False returns with __virtual__() + - **PR** `#42516`_: (*rallytime*) Add info about top file to pillar walk-through example to include edit.vim + - **PR** `#42479`_: (*gtmanfred*) validate ssh_interface for ec2 + - **PR** `#42433`_: (*terminalmage*) Only force saltenv/pillarenv to be a string when not None + | refs: `#42573`_ + - **PR** `#42414`_: (*vutny*) DOCS: unify hash sum with hash type format + - **PR** `#42387`_: (*DmitryKuzmenko*) Fix race condition in usage of weakvaluedict + - **PR** `#42339`_: (*isbm*) Bugfix: Jobs scheduled to run at a future time stay pending for Salt minions (bsc`#1036125`_) + - **PR** `#42077`_: (*vutny*) Fix scheduled job run on Master if `when` parameter is a list + | refs: `#42107`_ + - **PR** `#41973`_: (*vutny*) Fix Master/Minion scheduled jobs based on Cron expressions + | refs: `#42077`_ + - **PR** `#41968`_: (*root360-AndreasUlm*) Fix rabbitmqctl output sanitizer for version 3.6.10 + * e2dd443 Merge pull request `#42601`_ from rallytime/merge-2017.7 + * 36a1bcf Merge branch '2016.11' into '2017.7' + + * 4b16109 Merge pull request `#42339`_ from isbm/isbm-jobs-scheduled-in-a-future-bsc1036125 + + * bbba84c Bugfix: Jobs scheduled to run at a future time stay pending for Salt minions (bsc`#1036125`_) + + * 6c5a7c6 Merge pull request `#42077`_ from vutny/fix-jobs-scheduled-with-whens + + * b1960ce Fix scheduled job run on Master if `when` parameter is a list + + * f9cb536 Merge pull request `#42414`_ from vutny/unify-hash-params-format + + * d1f2a93 DOCS: unify hash sum with hash type format + + * 535c922 Merge pull request `#42523`_ from rallytime/`fix-42375`_ + + * 685c2cc Add information about returning a tuple with an error message + + * fa46651 Add a mention of the True/False returns with __virtual__() + + * 0df0e7e Merge pull request `#42527`_ from twangboy/win_wua + + * 0373791 Correct capatlization + + * af3bcc9 Document changes to Windows Update in 10/2016 + + * 69b0658 Merge pull request `#42551`_ from binocvlar/fix-lack-of-align-check-output + + * c4fabaa Remove '-s' (--script) argument to parted within align_check function + + * 9e0b4e9 Merge pull request `#42573`_ from rallytime/`bp-42433`_ + + * 0293429 Only force saltenv/pillarenv to be a string when not None + + * e931ed2 Merge pull request `#42571`_ from twangboy/win_add_pythonpath + + * d55a44d Avoid loading user site packages + + * 9af1eb2 Ignore any PYTHON* environment vars already on the system + + * 4e2fb03 Add pythonpath to batch files and service + + * de2f397 Merge pull request `#42387`_ from DSRCorporation/bugs/42371_KeyError_WeakValueDict + + * e721c7e Don't use `key in weakvaluedict` because it could lie. + + * 641a9d7 Merge pull request `#41968`_ from root360-AndreasUlm/fix-rabbitmqctl-output-handler + + * 76fd941 added tests for rabbitmq 3.6.10 output handler + + * 3602af1 Fix rabbitmqctl output handler for 3.6.10 + + * 66fede3 Merge pull request `#42479`_ from gtmanfred/interface + + * c32c1b2 fix pylint + + * 99ec634 validate ssh_interface for ec2 + + * a925c70 Merge pull request `#42516`_ from rallytime/`fix-42405`_ + + * e3a6717 Add info about top file to pillar walk-through example to include edit.vim + +- **PR** `#42290`_: (*isbm*) Backport of `#42270`_ + @ *2017-07-27T22:30:05Z* + + * 22eea38 Merge pull request `#42290`_ from isbm/isbm-module_run_parambug_42270_217 + * e38d432 Fix docs + + * 1e8a56e Describe function tagging + + * 1d72332 Describe function batching + + * 1391a05 Bugfix: syntax error in the example + + * 8c71257 Call unnamed parameters properly + + * 94c97a8 Update and correct the error message + + * ea83513 Bugfix: args gets ignored alongside named parameters + + * 74689e3 Add ability to use tagged functions in the same set + +- **PR** `#42251`_: (*twangboy*) Fix `unit.modules.test_win_ip` for Windows + @ *2017-07-27T19:22:03Z* + + * 4c20f1c Merge pull request `#42251`_ from twangboy/unit_win_test_win_ip + * 97261bf Fix win_inet_pton check for malformatted ip addresses + +- **PR** `#42255`_: (*twangboy*) Fix `unit.modules.test_win_system` for Windows + @ *2017-07-27T19:12:42Z* + + * 2985e4c Merge pull request `#42255`_ from twangboy/win_unit_test_win_system + * acc0345 Fix unit tests + +- **PR** `#42528`_: (*twangboy*) Namespace `cmp_to_key` in the pkg state for Windows + @ *2017-07-27T18:30:23Z* + + * a573386 Merge pull request `#42528`_ from twangboy/win_fix_pkg_state + * a040443 Move functools import inside pylint escapes + + * 118d513 Remove namespaced function `cmp_to_key` + + * a02c91a Namespace `cmp_to_key` in the pkg state for Windows + +- **PR** `#42534`_: (*jmarinaro*) Fixes AttributeError thrown by chocolatey state + @ *2017-07-27T17:59:50Z* + + - **ISSUE** `#42521`_: (*rickh563*) chocolatey.installed broken on 2017.7.0 + | refs: `#42534`_ + * 62ae12b Merge pull request `#42534`_ from jmarinaro/2017.7 + * b242d2d Fixes AttributeError thrown by chocolatey state Fixes `#42521`_ + +- **PR** `#42557`_: (*justincbeard*) Fixing output so --force-color and --no-color override master and min… + @ *2017-07-27T17:07:33Z* + + - **ISSUE** `#40354`_: (*exc414*) CentOS 6.8 Init Script - Sed unterminated address regex + | refs: `#42557`_ + - **ISSUE** `#37312`_: (*gtmanfred*) CLI flags should take overload settings in the config files + | refs: `#42557`_ + * 52605c2 Merge pull request `#42557`_ from justincbeard/bugfix_37312 + * ee3bc6e Fixing output so --force-color and --no-color override master and minion config color value + +- **PR** `#42567`_: (*skizunov*) Fix disable_ config option + @ *2017-07-27T17:05:00Z* + + * ab33517 Merge pull request `#42567`_ from skizunov/develop3 + * 0f0b7e3 Fix disable_ config option + +- **PR** `#42577`_: (*twangboy*) Compile scripts with -E -s params for Salt on Mac + @ *2017-07-26T22:44:37Z* + + * 30bb941 Merge pull request `#42577`_ from twangboy/mac_scripts + * 69d5973 Compile scripts with -E -s params for python + +- **PR** `#42524`_: (*rallytime*) [2017.7] Merge forward from 2016.11 to 2017.7 + @ *2017-07-26T22:41:06Z* + + - **ISSUE** `#42417`_: (*clem-compilatio*) salt-cloud - openstack - "no more floating IP addresses" error - but public_ip in node + | refs: `#42509`_ + - **ISSUE** `#42413`_: (*goten4*) Invalid error message when proxy_host is set and tornado not installed + | refs: `#42424`_ + - **ISSUE** `#42357`_: (*Giandom*) Salt pillarenv problem with slack engine + | refs: `#42443`_ `#42444`_ + - **ISSUE** `#42198`_: (*shengis*) state sqlite3.row_absent fail with "parameters are of unsupported type" + | refs: `#42200`_ + - **PR** `#42509`_: (*clem-compilatio*) Fix _assign_floating_ips in openstack.py + - **PR** `#42464`_: (*garethgreenaway*) [2016.11] Small fix to modules/git.py + - **PR** `#42443`_: (*garethgreenaway*) [2016.11] Fix to slack engine + - **PR** `#42424`_: (*goten4*) Fix error message when tornado or pycurl is not installed + - **PR** `#42200`_: (*shengis*) Fix `#42198`_ + * 60cd078 Merge pull request `#42524`_ from rallytime/merge-2017.7 + * 14d8d79 Merge branch '2016.11' into '2017.7' + + * 1bd5bbc Merge pull request `#42509`_ from clem-compilatio/`fix-42417`_ + + * 72924b0 Fix _assign_floating_ips in openstack.py + + * 4bf35a7 Merge pull request `#42464`_ from garethgreenaway/2016_11_remove_tmp_identity_file + + * ff24102 Uncomment the line that removes the temporary identity file. + + * e2120db Merge pull request `#42443`_ from garethgreenaway/42357_pass_args_kwargs_correctly + + * 635810b Updating the slack engine in 2016.11 to pass the args and kwrags correctly to LocalClient + + * 8262cc9 Merge pull request `#42200`_ from shengis/sqlite3_fix_row_absent_2016.11 + + * 407b8f4 Fix `#42198`_ If where_args is not set, not using it in the delete request. + + * d9df97e Merge pull request `#42424`_ from goten4/2016.11 + + * 1c0574d Fix error message when tornado or pycurl is not installed + +- **PR** `#42575`_: (*rallytime*) [2017.7] Merge forward from 2017.7.1 to 2017.7 + @ *2017-07-26T22:39:10Z* + + * 2acde83 Merge pull request `#42575`_ from rallytime/merge-2017.7.1-into-2017.7 + * 63bb0fb pass in empty kwarg for reactor + + * 2868061 update chunk, not kwarg in chunk + + * 46715e9 Merge branch '2017.7.1' into '2017.7' + +- **PR** `#42555`_: (*Ch3LL*) add changelog to 2017.7.1 release notes + @ *2017-07-26T14:57:43Z* + + * 1d93e92 Merge pull request `#42555`_ from Ch3LL/7.1_add_changelog + * fb69e71 add changelog to 2017.7.1 release notes + +- **PR** `#42266`_: (*twangboy*) Fix `unit.states.test_file` for Windows + @ *2017-07-25T20:26:32Z* + + * 07c2793 Merge pull request `#42266`_ from twangboy/win_unit_states_test_file + * 669aaee Mock file exists properly + + * a4231c9 Fix ret mock for linux + + * 0c484f8 Fix unit tests on Windows + +- **PR** `#42484`_: (*shengis*) Fix a potential Exception with an explicit error message + @ *2017-07-25T18:34:12Z* + + * df417ea Merge pull request `#42484`_ from shengis/fix-explicit-error-msg-x509-sign-remote + * 0b548c7 Fix a potential Exception with an explicit error message + +- **PR** `#42529`_: (*gtmanfred*) Fix joyent for python3 + @ *2017-07-25T16:37:48Z* + + - **ISSUE** `#41720`_: (*rallytime*) [Py3] Some salt-cloud drivers do not work using Python 3 + | refs: `#42529`_ + - **PR** `#396`_: (*mb0*) add file state template context and defaults + | refs: `#42529`_ + * 0f25ec7 Merge pull request `#42529`_ from gtmanfred/2017.7 + * b7ebb4d these drivers do not actually have an issue. + + * e90ca7a use salt encoding for joyent on 2017.7 + +- **PR** `#42465`_: (*garethgreenaway*) [2017.7] Small fix to modules/git.py + @ *2017-07-24T17:24:55Z* + + * 488457c Merge pull request `#42465`_ from garethgreenaway/2017_7_remove_tmp_identity_file + * 1920dc6 Uncomment the line that removes the temporary identity file. + +- **PR** `#42107`_: (*vutny*) [2017.7] Fix scheduled jobs if `when` parameter is a list + @ *2017-07-24T17:04:12Z* + + - **ISSUE** `#23516`_: (*dkiser*) BUG: cron job scheduler sporadically works + | refs: `#42077`_ + - **PR** `#42077`_: (*vutny*) Fix scheduled job run on Master if `when` parameter is a list + | refs: `#42107`_ + - **PR** `#41973`_: (*vutny*) Fix Master/Minion scheduled jobs based on Cron expressions + | refs: `#42077`_ + * 4f04499 Merge pull request `#42107`_ from vutny/2017.7-fix-jobs-scheduled-with-whens + * 905be49 [2017.7] Fix scheduled jobs if `when` parameter is a list + +- **PR** `#42506`_: (*terminalmage*) Add PER_REMOTE_ONLY to init_remotes call in git_pillar runner + @ *2017-07-24T16:59:21Z* + + * 6eaa076 Merge pull request `#42506`_ from terminalmage/fix-git-pillar-runner + * 6352f44 Add PER_REMOTE_ONLY to init_remotes call in git_pillar runner + +- **PR** `#42502`_: (*shengis*) Fix azurerm query to show IPs + @ *2017-07-24T15:54:45Z* + + * b88e645 Merge pull request `#42502`_ from shengis/fix_azurerm_request_ips + * 92f1890 Fix azurerm query to show IPs + +- **PR** `#42180`_: (*twangboy*) Fix `unit.modules.test_timezone` for Windows + @ *2017-07-24T14:46:16Z* + + * c793d83 Merge pull request `#42180`_ from twangboy/win_unit_test_timezone + * 832a3d8 Skip tests that use os.symlink on Windows + +- **PR** `#42474`_: (*whiteinge*) Cmd arg kwarg parsing test + @ *2017-07-24T14:13:30Z* + + - **PR** `#39646`_: (*terminalmage*) Handle deprecation of passing string args to load_args_and_kwargs + | refs: `#42474`_ + * 083ff00 Merge pull request `#42474`_ from whiteinge/cmd-arg-kwarg-parsing-test + * 0cc0c09 Lint fixes + + * 6609373 Add back support for string kwargs + + * 622ff5b Add LocalClient.cmd test for arg/kwarg parsing + + * 9f4eb80 Add a test.arg variant that cleans the pub kwargs by default + +- **PR** `#42425`_: (*rallytime*) [2017.7] Merge forward from 2016.11 to 2017.7 + @ *2017-07-21T22:43:41Z* + + - **ISSUE** `#42333`_: (*b3hni4*) Getting "invalid type of dict, a list is required" when trying to configure engines in master config file + | refs: `#42352`_ + - **ISSUE** `#32400`_: (*rallytime*) Document Default Config Values + | refs: `#42319`_ + - **PR** `#42370`_: (*rallytime*) [2016.11] Merge forward from 2016.3 to 2016.11 + - **PR** `#42368`_: (*twangboy*) Remove build and dist directories before install (2016.11) + - **PR** `#42360`_: (*Ch3LL*) [2016.11] Update version numbers in doc config for 2017.7.0 release + - **PR** `#42359`_: (*Ch3LL*) [2016.3] Update version numbers in doc config for 2017.7.0 release + - **PR** `#42356`_: (*meaksh*) Allow to check whether a function is available on the AliasesLoader wrapper + - **PR** `#42352`_: (*CorvinM*) Multiple documentation fixes + - **PR** `#42350`_: (*twangboy*) Fixes problem with Version and OS Release related grains on certain versions of Python (2016.11) + - **PR** `#42319`_: (*rallytime*) Add more documentation for config options that are missing from master/minion docs + * c91a5e5 Merge pull request `#42425`_ from rallytime/merge-2017.7 + * ea457aa Remove ALIASES block from template util + + * c673b64 Merge branch '2016.11' into '2017.7' + + * 42bb1a6 Merge pull request `#42350`_ from twangboy/win_fix_ver_grains_2016.11 + + * 8c04840 Detect Server OS with a desktop release name + + * 0a72e56 Merge pull request `#42356`_ from meaksh/2016.11-AliasesLoader-wrapper-fix + + * 915d942 Allow to check whether a function is available on the AliasesLoader wrapper + + * 10eb7b7 Merge pull request `#42368`_ from twangboy/win_fix_build_2016.11 + + * a7c910c Remove build and dist directories before install + + * 016189f Merge pull request `#42370`_ from rallytime/merge-2016.11 + + * 0aa5dde Merge branch '2016.3' into '2016.11' + + * e9b0f20 Merge pull request `#42359`_ from Ch3LL/doc-update-2016.3 + + * dc85b5e [2016.3] Update version numbers in doc config for 2017.7.0 release + + * f06a6f1 Merge pull request `#42360`_ from Ch3LL/doc-update-2016.11 + + * b90b7a7 [2016.11] Update version numbers in doc config for 2017.7.0 release + + * e0595b0 Merge pull request `#42319`_ from rallytime/config-docs + + * b40f980 Add more documentation for config options that are missing from master/minion docs + + * 7894040 Merge pull request `#42352`_ from CorvinM/issue42333 + + * 526b6ee Multiple documentation fixes + +- **PR** `#42444`_: (*garethgreenaway*) [2017.7] Fix to slack engine + @ *2017-07-21T22:03:48Z* + + - **ISSUE** `#42357`_: (*Giandom*) Salt pillarenv problem with slack engine + | refs: `#42443`_ `#42444`_ + * 10e4d92 Merge pull request `#42444`_ from garethgreenaway/42357_2017_7_pass_args_kwargs_correctly + * f411cfc Updating the slack engine in 2017.7 to pass the args and kwrags correctly to LocalClient + +- **PR** `#42461`_: (*rallytime*) Bump warning version from Oxygen to Fluorine in roster cache + @ *2017-07-21T21:33:25Z* + + * 723be49 Merge pull request `#42461`_ from rallytime/bump-roster-cache-deprecations + * c0df013 Bump warning version from Oxygen to Fluorine in roster cache + +- **PR** `#42436`_: (*garethgreenaway*) Fixes to versions function in manage runner + @ *2017-07-21T19:41:07Z* + + - **ISSUE** `#42374`_: (*tyhunt99*) [2017.7.0] salt-run mange.versions throws exception if minion is offline or unresponsive + | refs: `#42436`_ + * 0952160 Merge pull request `#42436`_ from garethgreenaway/42374_manage_runner_minion_offline + * 0fd3949 Updating the versions function inside the manage runner to account for when a minion is offline and we are unable to determine it's version. + +- **PR** `#42435`_: (*terminalmage*) Modify our custom YAML loader to treat unicode literals as unicode strings + | refs: `#42812`_ + @ *2017-07-21T19:40:34Z* + + - **ISSUE** `#42427`_: (*grichmond-salt*) Issue Passing Variables created from load_json as Inline Pillar Between States + | refs: `#42435`_ + * 54193ea Merge pull request `#42435`_ from terminalmage/issue42427 + * 31273c7 Modify our custom YAML loader to treat unicode literals as unicode strings + +- **PR** `#42399`_: (*rallytime*) Update old "ref" references to "rev" in git.detached state + @ *2017-07-21T19:38:59Z* + + - **ISSUE** `#42381`_: (*zebooka*) Git.detached broken in 2017.7.0 + | refs: `#42399`_ + - **ISSUE** `#38878`_: (*tomlaredo*) [Naming consistency] git.latest "rev" option VS git.detached "ref" option + | refs: `#38898`_ + - **PR** `#38898`_: (*terminalmage*) git.detached: rename ref to rev for consistency + | refs: `#42399`_ + * 0b31791 Merge pull request `#42399`_ from rallytime/`fix-42381`_ + * d9d94fe Update old "ref" references to "rev" in git.detached state + +- **PR** `#42031`_: (*skizunov*) Fix: Reactor emits critical error + @ *2017-07-21T19:38:34Z* + + - **ISSUE** `#42400`_: (*Enquier*) Conflict in execution of passing pillar data to orch/reactor event executions 2017.7.0 + | refs: `#42031`_ + * bd4adb4 Merge pull request `#42031`_ from skizunov/develop3 + * 540977b Fix: Reactor emits critical error + +- **PR** `#42027`_: (*gtmanfred*) import salt.minion for EventReturn for Windows + @ *2017-07-21T19:37:03Z* + + - **ISSUE** `#41949`_: (*jrporcaro*) Event returner doesn't work with Windows Master + | refs: `#42027`_ + * 3abf7ad Merge pull request `#42027`_ from gtmanfred/2017.7 + * fd4458b import salt.minion for EventReturn for Windows + +- **PR** `#42454`_: (*terminalmage*) Document future renaming of new rand_str jinja filter + @ *2017-07-21T18:47:51Z* + + * 994d3dc Merge pull request `#42454`_ from terminalmage/jinja-docs-2017.7 + * 98b6614 Document future renaming of new rand_str jinja filter + +- **PR** `#42452`_: (*Ch3LL*) update windows urls to new py2/py3 naming scheme + @ *2017-07-21T17:20:47Z* + + * 4480075 Merge pull request `#42452`_ from Ch3LL/fix_url_windows + * 3f4a918 update windows urls to new py2/py3 naming scheme + +- **PR** `#42411`_: (*seedickcode*) Fix file.managed check_cmd file not found - Issue `#42404`_ + @ *2017-07-20T21:59:17Z* + + - **ISSUE** `#42404`_: (*gabekahen*) [2017.7] file.managed with cmd_check "No such file or directory" + | refs: `#42411`_ + - **ISSUE** `#33708`_: (*pepinje*) visudo check command leaves cache file in /tmp + | refs: `#42411`_ `#38063`_ + - **PR** `#38063`_: (*llua*) tmp file clean up in file.manage - fix for `#33708`_ + | refs: `#42411`_ + * 33e90be Merge pull request `#42411`_ from seedickcode/check_cmd_fix + * 4ae3911 Fix file.managed check_cmd file not found - Issue `#42404`_ + +- **PR** `#42409`_: (*twangboy*) Add Scripts to build Py3 on Mac + @ *2017-07-20T21:36:34Z* + + * edde313 Merge pull request `#42409`_ from twangboy/mac_py3_scripts + * ac0e04a Remove build and dist, sign pkgs + + * 9d66e27 Fix hard coded pip path + + * 7b8d6cb Add support for Py3 + + * aa4eed9 Update Python and other reqs + +- **PR** `#42433`_: (*terminalmage*) Only force saltenv/pillarenv to be a string when not None + | refs: `#42573`_ + @ *2017-07-20T21:32:24Z* + + - **ISSUE** `#42403`_: (*astronouth7303*) [2017.7] Pillar empty when state is applied from orchestrate + | refs: `#42433`_ + * 82982f9 Merge pull request `#42433`_ from terminalmage/issue42403 +- **PR** `#42408`_: (*CorvinM*) Fix documentation misformat in salt.states.file.replace + @ *2017-07-20T00:45:43Z* + + * a71938c Merge pull request `#42408`_ from CorvinM/file-replace-doc-fix + * 246a2b3 Fix documentation misformat in salt.states.file.replace + +- **PR** `#42347`_: (*twangboy*) Fixes problem with Version and OS Release related grains on certain versions of Python + @ *2017-07-19T17:05:43Z* + + * d385dfd Merge pull request `#42347`_ from twangboy/win_fix_ver_grains + * ef1f663 Detect server OS with a desktop release name + +- **PR** `#42366`_: (*twangboy*) Remove build and dist directories before install + @ *2017-07-19T16:37:41Z* + + * eb9e420 Merge pull request `#42366`_ from twangboy/win_fix_build + * 0946002 Add blank line after delete + + * f7c0bb4 Remove build and dist directories before install + +- **PR** `#42373`_: (*Ch3LL*) Add initial 2017.7.1 Release Notes File + @ *2017-07-19T16:28:46Z* + + * af7820f Merge pull request `#42373`_ from Ch3LL/add_2017.7.1 + * ce1c1b6 Add initial 2017.7.1 Release Notes File + +- **PR** `#42150`_: (*twangboy*) Fix `unit.modules.test_pip` for Windows + @ *2017-07-19T16:01:17Z* + + * 59e012b Merge pull request `#42150`_ from twangboy/win_unit_test_pip + * 4ee2420 Fix unit tests for test_pip + +- **PR** `#42154`_: (*twangboy*) Fix `unit.modules.test_reg_win` for Windows + @ *2017-07-19T16:00:38Z* + + * ade25c6 Merge pull request `#42154`_ from twangboy/win_unit_test_reg + * 00d9a52 Fix problem with handling REG_QWORD in list values + +- **PR** `#42182`_: (*twangboy*) Fix `unit.modules.test_useradd` for Windows + @ *2017-07-19T15:55:33Z* + + * 0759367 Merge pull request `#42182`_ from twangboy/win_unit_test_useradd + * 8260a71 Disable tests that require pwd in Windows + +- **PR** `#42364`_: (*twangboy*) Windows Package notes for 2017.7.0 + @ *2017-07-18T19:24:45Z* + + * a175c40 Merge pull request `#42364`_ from twangboy/release_notes_2017.7.0 + * 96517d1 Add note about patched windows packages + +- **PR** `#42361`_: (*Ch3LL*) [2017.7] Update version numbers in doc config for 2017.7.0 release + @ *2017-07-18T19:23:22Z* + + * 4dfe50e Merge pull request `#42361`_ from Ch3LL/doc-update-2017.7 + * dc5bb30 [2017.7] Update version numbers in doc config for 2017.7.0 release + +- **PR** `#42363`_: (*rallytime*) [2017.7] Merge forward from 2016.11 to 2017.7 + @ *2017-07-18T18:40:48Z* + + - **ISSUE** `#42295`_: (*lubyou*) file.absent fails on windows if the file to be removed has the "readonly" attribute set + | refs: `#42308`_ + - **ISSUE** `#42267`_: (*gzcwnk*) salt-ssh not creating ssh keys automatically as per documentation + | refs: `#42314`_ + - **ISSUE** `#42240`_: (*casselt*) empty_password in user.present always changes password, even with test=True + | refs: `#42289`_ + - **ISSUE** `#42232`_: (*astronouth7303*) Half of dnsutil refers to dig + | refs: `#42235`_ + - **ISSUE** `#42194`_: (*jryberg*) pkg version: latest are now broken, appending -latest to filename + | refs: `#42275`_ + - **ISSUE** `#42152`_: (*dubb-b*) salt-cloud errors on Rackspace driver using -out=yaml + | refs: `#42282`_ + - **ISSUE** `#42137`_: (*kiemlicz*) cmd.run with multiple commands - random order of execution + | refs: `#42181`_ + - **ISSUE** `#42116`_: (*terminalmage*) CLI pillar override regression in 2017.7.0rc1 + | refs: `#42119`_ + - **ISSUE** `#42115`_: (*nomeelnoj*) Installing EPEL repo breaks salt-cloud + | refs: `#42163`_ + - **ISSUE** `#42114`_: (*clallen*) saltenv bug in pillar.get execution module function + | refs: `#42121`_ + - **ISSUE** `#41936`_: (*michaelkarrer81*) git.latest identity does not set the correct user for the private key file on the minion + | refs: `#41945`_ + - **ISSUE** `#41721`_: (*sazaro*) state.sysrc broken when setting the value to YES or NO + | refs: `#42269`_ + - **ISSUE** `#41116`_: (*hrumph*) FAQ has wrong instructions for upgrading Windows minion. + | refs: `#42264`_ + - **ISSUE** `#39365`_: (*dglloyd*) service.running fails if sysv script has no status command and enable: True + | refs: `#39366`_ + - **ISSUE** `#1`_: (*thatch45*) Enable regex on the salt cli + - **PR** `#42353`_: (*terminalmage*) is_windows is a function, not a propery/attribute + - **PR** `#42314`_: (*rallytime*) Add clarification to salt ssh docs about key auto-generation. + - **PR** `#42308`_: (*lubyou*) Force file removal on Windows. Fixes `#42295`_ + - **PR** `#42289`_: (*CorvinM*) Multiple empty_password fixes for state.user + - **PR** `#42282`_: (*rallytime*) Handle libcloud objects that throw RepresenterErrors with --out=yaml + - **PR** `#42275`_: (*terminalmage*) pkg.installed: pack name/version into pkgs argument + - **PR** `#42269`_: (*rallytime*) Add some clarity to "multiple quotes" section of yaml docs + - **PR** `#42264`_: (*rallytime*) Update minion restart section in FAQ doc for windows + - **PR** `#42262`_: (*rallytime*) Back-port `#42224`_ to 2016.11 + - **PR** `#42261`_: (*rallytime*) Some minor doc fixes for dnsutil module so they'll render correctly + - **PR** `#42253`_: (*gtmanfred*) Only use unassociated ips when unable to allocate + - **PR** `#42252`_: (*UtahDave*) simple docstring updates + - **PR** `#42235`_: (*astronouth7303*) Abolish references to `dig` in examples. + - **PR** `#42224`_: (*tdutrion*) Remove duplicate instruction in Openstack Rackspace config example + | refs: `#42262`_ + - **PR** `#42215`_: (*twangboy*) Add missing config to example + - **PR** `#42211`_: (*terminalmage*) Only pass a saltenv in orchestration if one was explicitly passed (2016.11) + - **PR** `#42181`_: (*garethgreenaway*) fixes to state.py for names parameter + - **PR** `#42176`_: (*rallytime*) Back-port `#42109`_ to 2016.11 + - **PR** `#42175`_: (*rallytime*) Back-port `#39366`_ to 2016.11 + - **PR** `#42173`_: (*rallytime*) Back-port `#37424`_ to 2016.11 + - **PR** `#42172`_: (*rallytime*) [2016.11] Merge forward from 2016.3 to 2016.11 + - **PR** `#42164`_: (*Ch3LL*) Fix kerberos create_keytab doc + - **PR** `#42163`_: (*vutny*) Fix `#42115`_: parse libcloud "rc" version correctly + - **PR** `#42155`_: (*phsteve*) Fix docs for puppet.plugin_sync + - **PR** `#42142`_: (*Ch3LL*) Update builds available for rc1 + - **PR** `#42141`_: (*rallytime*) Back-port `#42098`_ to 2016.11 + - **PR** `#42140`_: (*rallytime*) Back-port `#42097`_ to 2016.11 + - **PR** `#42123`_: (*vutny*) DOCS: describe importing custom util classes + - **PR** `#42121`_: (*terminalmage*) Fix pillar.get when saltenv is passed + - **PR** `#42119`_: (*terminalmage*) Fix regression in CLI pillar override for salt-call + - **PR** `#42109`_: (*arthurlogilab*) [doc] Update aws.rst - add Debian default username + | refs: `#42176`_ + - **PR** `#42098`_: (*twangboy*) Change repo_ng to repo-ng + | refs: `#42141`_ + - **PR** `#42097`_: (*gtmanfred*) require large timediff for ipv6 warning + | refs: `#42140`_ + - **PR** `#42095`_: (*terminalmage*) Add debug logging to dockerng.login + - **PR** `#42094`_: (*terminalmage*) Prevent command from showing in exception when output_loglevel=quiet + - **PR** `#41945`_: (*garethgreenaway*) Fixes to modules/git.py + - **PR** `#41543`_: (*cri-epita*) Fix user creation with empty password + | refs: `#42289`_ `#42289`_ + - **PR** `#39366`_: (*dglloyd*) Pass sig to service.status in after_toggle + | refs: `#42175`_ + - **PR** `#38965`_: (*toanju*) salt-cloud will use list_floating_ips for OpenStack + | refs: `#42253`_ + - **PR** `#37424`_: (*kojiromike*) Avoid Early Convert ret['comment'] to String + | refs: `#42173`_ + - **PR** `#34280`_: (*kevinanderson1*) salt-cloud will use list_floating_ips for Openstack + | refs: `#38965`_ + * 587138d Merge pull request `#42363`_ from rallytime/merge-2017.7 + * 7aa31ff Merge branch '2016.11' into '2017.7' + + * b256001 Merge pull request `#42353`_ from terminalmage/fix-git-test + + * 14cf6ce is_windows is a function, not a propery/attribute + + * 866a1fe Merge pull request `#42264`_ from rallytime/`fix-41116`_ + + * bd63888 Add mono-spacing to salt-minion reference for consistency + + * 30d62f4 Update minion restart section in FAQ doc for windows + + * 9a70708 Merge pull request `#42275`_ from terminalmage/issue42194 + + * 6638749 pkg.installed: pack name/version into pkgs argument + + * e588f23 Merge pull request `#42269`_ from rallytime/`fix-41721`_ + + * f2250d4 Add a note about using different styles of quotes. + + * 38d9b3d Add some clarity to "multiple quotes" section of yaml docs + + * 5aaa214 Merge pull request `#42282`_ from rallytime/`fix-42152`_ + + * f032223 Handle libcloud objects that throw RepresenterErrors with --out=yaml + + * fb5697a Merge pull request `#42308`_ from lubyou/42295-fix-file-absent-windows + + * 026ccf4 Force file removal on Windows. Fixes `#42295`_ + + * da2a8a5 Merge pull request `#42314`_ from rallytime/`fix-42267`_ + + * c406046 Add clarification to salt ssh docs about key auto-generation. + + * acadd54 Merge pull request `#41945`_ from garethgreenaway/41936_allow_identity_files_with_user + + * 44841e5 Moving the call to cp.get_file inside the with block to ensure the umask is preserved when we grab the file. + + * f9ba60e Merge pull request `#1`_ from terminalmage/pr-41945 + + * 1b60261 Restrict set_umask to mkstemp call only + + * 68549f3 Fixing umask to we can set files as executable. + + * 4949bf3 Updating to swap on the new salt.utils.files.set_umask context_manager + + * 8faa9f6 Updating PR with requested changes. + + * 494765e Updating the git module to allow an identity file to be used when passing the user parameter + + * f90e04a Merge pull request `#42289`_ from CorvinM/`bp-41543`_ + + * 357dc22 Fix user creation with empty password + + * a91a3f8 Merge pull request `#42123`_ from vutny/fix-master-utils-import + + * 6bb8b8f Add missing doc for ``utils_dirs`` Minion config option + + * f1bc58f Utils: add example of module import + + * e2aa511 Merge pull request `#42261`_ from rallytime/minor-doc-fix + + * 8c76bbb Some minor doc fixes for dnsutil module so they'll render correctly + + * 3e9dfbc Merge pull request `#42262`_ from rallytime/`bp-42224`_ + + * c31ded3 Remove duplicate instruction in Openstack Rackspace config example + + * 7780579 Merge pull request `#42181`_ from garethgreenaway/42137_backport_fix_from_2017_7 + + * a34970b Back porting the fix for 2017.7 that ensures the order of the names parameter. + + * 7253786 Merge pull request `#42253`_ from gtmanfred/2016.11 + + * 53e2576 Only use unassociated ips when unable to allocate + + * b2a4698 Merge pull request `#42252`_ from UtahDave/2016.11local + + * e6a9563 simple doc updates + + * 781fe13 Merge pull request `#42235`_ from astronouth7303/patch-1-2016.3 + + * 4cb51bd Make note of dig partial requirement. + + * 08e7d83 Abolish references to `dig` in examples. + + * 83cbd76 Merge pull request `#42215`_ from twangboy/win_iis_docs + + * c07e220 Add missing config to example + + * 274946a Merge pull request `#42211`_ from terminalmage/issue40928 + + * 22a18fa Only pass a saltenv in orchestration if one was explicitly passed (2016.11) + + * 89261cf Merge pull request `#42173`_ from rallytime/`bp-37424`_ + + * 01addb6 Avoid Early Convert ret['comment'] to String + + * 3b17fb7 Merge pull request `#42175`_ from rallytime/`bp-39366`_ + + * 53f7b98 Pass sig to service.status in after_toggle + + * ea16f47 Merge pull request `#42172`_ from rallytime/merge-2016.11 + + * b1fa332 Merge branch '2016.3' into '2016.11' + + * 8fa1fa5 Merge pull request `#42155`_ from phsteve/doc-fix-puppet + + * fb2cb78 Fix docs for puppet.plugin_sync so code-block renders properly and sync is spelled consistently + + * 6307b98 Merge pull request `#42176`_ from rallytime/`bp-42109`_ + + * 686926d Update aws.rst - add Debian default username + + * 28c4e4c Merge pull request `#42095`_ from terminalmage/docker-login-debugging + + * bd27870 Add debug logging to dockerng.login + + * 2b754bc Merge pull request `#42119`_ from terminalmage/issue42116 + + * 9a26894 Add integration test for 42116 + + * 1bb42bb Fix regression when CLI pillar override is used with salt-call + + * 8c0a83c Merge pull request `#42121`_ from terminalmage/issue42114 + + * d142912 Fix pillar.get when saltenv is passed + + * 687992c Merge pull request `#42094`_ from terminalmage/quiet-exception + + * 47d61f4 Prevent command from showing in exception when output_loglevel=quiet + + * dad2551 Merge pull request `#42163`_ from vutny/`fix-42115`_ + + * b27b1e3 Fix `#42115`_: parse libcloud "rc" version correctly + + * 2a8ae2b Merge pull request `#42164`_ from Ch3LL/fix_kerb_doc + + * 7c0fb24 Fix kerberos create_keytab doc + + * 678d4d4 Merge pull request `#42141`_ from rallytime/`bp-42098`_ + + * bd80243 Change repo_ng to repo-ng + + * c8afd7a Merge pull request `#42140`_ from rallytime/`bp-42097`_ + + * 9c4e132 Import datetime + + * 1435bf1 require large timediff for ipv6 warning + + * c239664 Merge pull request `#42142`_ from Ch3LL/change_builds + + * e1694af Update builds available for rc1 + +- **PR** `#42340`_: (*isbm*) Bugfix: Jobs scheduled to run at a future time stay pending for Salt … + @ *2017-07-18T18:13:36Z* + + - **ISSUE** `#1036125`_: (**) + * 55b7a5c Merge pull request `#42340`_ from isbm/isbm-jobs-scheduled-in-a-future-2017.7-bsc1036125 + * 774d204 Bugfix: Jobs scheduled to run at a future time stay pending for Salt minions (bsc`#1036125`_) + +- **PR** `#42327`_: (*mirceaulinic*) Default skip_verify to False + @ *2017-07-18T18:04:36Z* + + * e72616c Merge pull request `#42327`_ from mirceaulinic/patch-10 + * c830573 Trailing whitespaces + + * c83e6fc Default skip_verify to False + +- **PR** `#42179`_: (*rallytime*) Fix some documentation issues found in jinja filters doc topic + @ *2017-07-18T18:01:57Z* + + - **ISSUE** `#42151`_: (*sjorge*) Doc errors in jinja doc for develop branch + | refs: `#42179`_ `#42179`_ + * ba799b2 Merge pull request `#42179`_ from rallytime/`fix-42151`_ + * 798d292 Add note about "to_bytes" jinja filter issues when using yaml_jinja renderer + + * 1bbff57 Fix some documentation issues found in jinja filters doc topic + +- **PR** `#42087`_: (*abulford*) Make result=true if Docker volume already exists + @ *2017-07-17T18:41:47Z* + + - **ISSUE** `#42076`_: (*abulford*) dockerng.volume_present test looks as though it would cause a change + | refs: `#42086`_ `#42086`_ `#42087`_ `#42087`_ + - **PR** `#42086`_: (*abulford*) Make result=true if Docker volume already exists + | refs: `#42087`_ + * 8dbb938 Merge pull request `#42087`_ from redmatter/fix-dockerng-volume-present-result-2017.7 + * 2e1dc95 Make result=true if Docker volume already exists + +- **PR** `#42186`_: (*rallytime*) Use long_range function for IPv6Network hosts() function + @ *2017-07-17T18:39:35Z* + + - **ISSUE** `#42166`_: (*sjorge*) [2017.7.0rc1] jinja filter network_hosts fails on large IPv6 networks + | refs: `#42186`_ + * c84d6db Merge pull request `#42186`_ from rallytime/`fix-42166`_ + * b8bcc0d Add note to various network_hosts docs about long_run for IPv6 networks + + * 1186274 Use long_range function for IPv6Network hosts() function + +- **PR** `#42210`_: (*terminalmage*) Only pass a saltenv in orchestration if one was explicitly passed (2017.7) + @ *2017-07-17T18:22:39Z* + + * e7b79e0 Merge pull request `#42210`_ from terminalmage/issue40928-2017.7 + * 771ade5 Only pass a saltenv in orchestration if one was explicitly passed (2017.7) + +- **PR** `#42236`_: (*mirceaulinic*) New option for napalm proxy/minion: provider + @ *2017-07-17T18:19:56Z* + + * 0e49021 Merge pull request `#42236`_ from cloudflare/napalm-provider + * 1ac69bd Document the provider option and rearrange the doc + + * 4bf4b14 New option for napalm proxy/minion: provider + +- **PR** `#42257`_: (*twangboy*) Fix `unit.pillar.test_git` for Windows + @ *2017-07-17T17:51:42Z* + + * 3ec5bb1 Merge pull request `#42257`_ from twangboy/win_unit_pillar_test_git + * 45be326 Add error-handling function to shutil.rmtree + +- **PR** `#42258`_: (*twangboy*) Fix `unit.states.test_environ` for Windows + @ *2017-07-17T17:50:38Z* + + * 3639562 Merge pull request `#42258`_ from twangboy/win_unit_states_tests_environ + * 55b278c Mock the reg.read_value function + +- **PR** `#42265`_: (*rallytime*) Gate boto_elb tests if proper version of moto isn't installed + @ *2017-07-17T17:47:52Z* + + * 894bdd2 Merge pull request `#42265`_ from rallytime/gate-moto-version + * 78cdee5 Gate boto_elb tests if proper version of moto isn't installed + +- **PR** `#42277`_: (*twangboy*) Fix `unit.states.test_winrepo` for Windows + @ *2017-07-17T17:37:07Z* + + * baf04f2 Merge pull request `#42277`_ from twangboy/win_unit_states_test_winrepo + * ed89cd0 Use os.sep for path seps + +- **PR** `#42309`_: (*terminalmage*) Change "TBD" in versionadded to "2017.7.0" + @ *2017-07-17T17:11:45Z* + + * be6b211 Merge pull request `#42309`_ from terminalmage/fix-versionadded + * 603f5b7 Change "TBD" in versionadded to "2017.7.0" + +- **PR** `#42206`_: (*rallytime*) [PY3] Fix test that is flaky in Python 3 + | refs: `#42783`_ + @ *2017-07-17T17:09:53Z* + + * acd29f9 Merge pull request `#42206`_ from rallytime/fix-flaky-test + * 2be4865 [PY3] Fix test that is flaky in Python 3 + +- **PR** `#42126`_: (*rallytime*) [2017.7] Merge forward from 2016.11 to 2017.7 + @ *2017-07-17T17:07:19Z* + + * 8f1cb28 Merge pull request `#42126`_ from rallytime/merge-2017.7 +* 8b35b36 Merge branch '2016.11' into '2017.7' + + +- **PR** `#42078`_: (*damon-atkins*) pkg.install and pkg.remove fix version number input. + @ *2017-07-05T06:04:57Z* + + * 4780d78 Merge pull request `#42078`_ from damon-atkins/fix_convert_flt_str_version_on_cmd_line + * 09d37dd Fix comment typo + + * 7167549 Handle version=None when converted to a string it becomes 'None' parm should default to empty string rather than None, it would fix better with existing code. + + * 4fb2bb1 Fix typo + + * cf55c33 pkg.install and pkg.remove on the command line take number version numbers, store them within a float. However version is a string, to support versions numbers like 1.3.4 + +- **PR** `#42105`_: (*Ch3LL*) Update releasecanddiate doc with new 2017.7.0rc1 Release + @ *2017-07-04T03:14:42Z* + + * 46d575a Merge pull request `#42105`_ from Ch3LL/update_rc + * d4e7b91 Update releasecanddiate doc with new 2017.7.0rc1 Release + +- **PR** `#42099`_: (*rallytime*) Remove references in docs to pip install salt-cloud + @ *2017-07-03T22:13:44Z* + + - **ISSUE** `#41885`_: (*astronouth7303*) Recommended pip installation outdated? + | refs: `#42099`_ + * d38548b Merge pull request `#42099`_ from rallytime/`fix-41885`_ + * c2822e0 Remove references in docs to pip install salt-cloud + +- **PR** `#42086`_: (*abulford*) Make result=true if Docker volume already exists + | refs: `#42087`_ + @ *2017-07-03T15:48:33Z* + + - **ISSUE** `#42076`_: (*abulford*) dockerng.volume_present test looks as though it would cause a change + | refs: `#42086`_ `#42086`_ `#42087`_ `#42087`_ + * 81d606a Merge pull request `#42086`_ from redmatter/fix-dockerng-volume-present-result + * 8d54968 Make result=true if Docker volume already exists + +- **PR** `#42021`_: (*gtmanfred*) Set concurrent to True when running states with sudo + @ *2017-06-30T21:02:15Z* + + - **ISSUE** `#25842`_: (*shikhartanwar*) Running salt-minion as non-root user to execute sudo commands always returns an error + | refs: `#42021`_ + * 7160697 Merge pull request `#42021`_ from gtmanfred/2016.11 + * 26beb18 Set concurrent to True when running states with sudo + +- **PR** `#42029`_: (*terminalmage*) Mock socket.getaddrinfo in unit.utils.network_test.NetworkTestCase.test_host_to_ips + @ *2017-06-30T20:58:56Z* + + * b784fbb Merge pull request `#42029`_ from terminalmage/host_to_ips + * 26f848e Mock socket.getaddrinfo in unit.utils.network_test.NetworkTestCase.test_host_to_ips + +- **PR** `#42055`_: (*dmurphy18*) Upgrade support for gnupg v2.1 and higher + @ *2017-06-30T20:54:02Z* + + * e067020 Merge pull request `#42055`_ from dmurphy18/handle_gnupgv21 + * e20cea6 Upgrade support for gnupg v2.1 and higher + +- **PR** `#42048`_: (*Ch3LL*) Add initial 2016.11.7 Release Notes + @ *2017-06-30T16:00:05Z* + + * 74ba2ab Merge pull request `#42048`_ from Ch3LL/add_11.7 + * 1de5e00 Add initial 2016.11.7 Release Notes + + +.. _`#1`: https://github.com/saltstack/salt/issues/1 +.. _`#1036125`: https://github.com/saltstack/salt/issues/1036125 +.. _`#12587`: https://github.com/saltstack/salt/issues/12587 +.. _`#15171`: https://github.com/saltstack/salt/issues/15171 +.. _`#2`: https://github.com/saltstack/salt/issues/2 +.. _`#23516`: https://github.com/saltstack/salt/issues/23516 +.. _`#25842`: https://github.com/saltstack/salt/issues/25842 +.. _`#26995`: https://github.com/saltstack/salt/issues/26995 +.. _`#32400`: https://github.com/saltstack/salt/issues/32400 +.. _`#33708`: https://github.com/saltstack/salt/issues/33708 +.. _`#33806`: https://github.com/saltstack/salt/pull/33806 +.. _`#34245`: https://github.com/saltstack/salt/issues/34245 +.. _`#34280`: https://github.com/saltstack/salt/pull/34280 +.. _`#37312`: https://github.com/saltstack/salt/issues/37312 +.. _`#37424`: https://github.com/saltstack/salt/pull/37424 +.. _`#38063`: https://github.com/saltstack/salt/pull/38063 +.. _`#38839`: https://github.com/saltstack/salt/issues/38839 +.. _`#38878`: https://github.com/saltstack/salt/issues/38878 +.. _`#38898`: https://github.com/saltstack/salt/pull/38898 +.. _`#38965`: https://github.com/saltstack/salt/pull/38965 +.. _`#39365`: https://github.com/saltstack/salt/issues/39365 +.. _`#39366`: https://github.com/saltstack/salt/pull/39366 +.. _`#39516`: https://github.com/saltstack/salt/pull/39516 +.. _`#396`: https://github.com/saltstack/salt/pull/396 +.. _`#39646`: https://github.com/saltstack/salt/pull/39646 +.. _`#39773`: https://github.com/saltstack/salt/pull/39773 +.. _`#40354`: https://github.com/saltstack/salt/issues/40354 +.. _`#40490`: https://github.com/saltstack/salt/issues/40490 +.. _`#41116`: https://github.com/saltstack/salt/issues/41116 +.. _`#41433`: https://github.com/saltstack/salt/issues/41433 +.. _`#41543`: https://github.com/saltstack/salt/pull/41543 +.. _`#41690`: https://github.com/saltstack/salt/pull/41690 +.. _`#41720`: https://github.com/saltstack/salt/issues/41720 +.. _`#41721`: https://github.com/saltstack/salt/issues/41721 +.. _`#41770`: https://github.com/saltstack/salt/issues/41770 +.. _`#41885`: https://github.com/saltstack/salt/issues/41885 +.. _`#41936`: https://github.com/saltstack/salt/issues/41936 +.. _`#41945`: https://github.com/saltstack/salt/pull/41945 +.. _`#41949`: https://github.com/saltstack/salt/issues/41949 +.. _`#41955`: https://github.com/saltstack/salt/issues/41955 +.. _`#41968`: https://github.com/saltstack/salt/pull/41968 +.. _`#41973`: https://github.com/saltstack/salt/pull/41973 +.. _`#41976`: https://github.com/saltstack/salt/issues/41976 +.. _`#41977`: https://github.com/saltstack/salt/pull/41977 +.. _`#41982`: https://github.com/saltstack/salt/issues/41982 +.. _`#41988`: https://github.com/saltstack/salt/pull/41988 +.. _`#41994`: https://github.com/saltstack/salt/pull/41994 +.. _`#42006`: https://github.com/saltstack/salt/pull/42006 +.. _`#42021`: https://github.com/saltstack/salt/pull/42021 +.. _`#42027`: https://github.com/saltstack/salt/pull/42027 +.. _`#42029`: https://github.com/saltstack/salt/pull/42029 +.. _`#42031`: https://github.com/saltstack/salt/pull/42031 +.. _`#42041`: https://github.com/saltstack/salt/issues/42041 +.. _`#42045`: https://github.com/saltstack/salt/pull/42045 +.. _`#42048`: https://github.com/saltstack/salt/pull/42048 +.. _`#42055`: https://github.com/saltstack/salt/pull/42055 +.. _`#42067`: https://github.com/saltstack/salt/pull/42067 +.. _`#42076`: https://github.com/saltstack/salt/issues/42076 +.. _`#42077`: https://github.com/saltstack/salt/pull/42077 +.. _`#42078`: https://github.com/saltstack/salt/pull/42078 +.. _`#42086`: https://github.com/saltstack/salt/pull/42086 +.. _`#42087`: https://github.com/saltstack/salt/pull/42087 +.. _`#42094`: https://github.com/saltstack/salt/pull/42094 +.. _`#42095`: https://github.com/saltstack/salt/pull/42095 +.. _`#42097`: https://github.com/saltstack/salt/pull/42097 +.. _`#42098`: https://github.com/saltstack/salt/pull/42098 +.. _`#42099`: https://github.com/saltstack/salt/pull/42099 +.. _`#42105`: https://github.com/saltstack/salt/pull/42105 +.. _`#42107`: https://github.com/saltstack/salt/pull/42107 +.. _`#42109`: https://github.com/saltstack/salt/pull/42109 +.. _`#42114`: https://github.com/saltstack/salt/issues/42114 +.. _`#42115`: https://github.com/saltstack/salt/issues/42115 +.. _`#42116`: https://github.com/saltstack/salt/issues/42116 +.. _`#42119`: https://github.com/saltstack/salt/pull/42119 +.. _`#42121`: https://github.com/saltstack/salt/pull/42121 +.. _`#42123`: https://github.com/saltstack/salt/pull/42123 +.. _`#42126`: https://github.com/saltstack/salt/pull/42126 +.. _`#42137`: https://github.com/saltstack/salt/issues/42137 +.. _`#42140`: https://github.com/saltstack/salt/pull/42140 +.. _`#42141`: https://github.com/saltstack/salt/pull/42141 +.. _`#42142`: https://github.com/saltstack/salt/pull/42142 +.. _`#42150`: https://github.com/saltstack/salt/pull/42150 +.. _`#42151`: https://github.com/saltstack/salt/issues/42151 +.. _`#42152`: https://github.com/saltstack/salt/issues/42152 +.. _`#42154`: https://github.com/saltstack/salt/pull/42154 +.. _`#42155`: https://github.com/saltstack/salt/pull/42155 +.. _`#42163`: https://github.com/saltstack/salt/pull/42163 +.. _`#42164`: https://github.com/saltstack/salt/pull/42164 +.. _`#42166`: https://github.com/saltstack/salt/issues/42166 +.. _`#42172`: https://github.com/saltstack/salt/pull/42172 +.. _`#42173`: https://github.com/saltstack/salt/pull/42173 +.. _`#42174`: https://github.com/saltstack/salt/pull/42174 +.. _`#42175`: https://github.com/saltstack/salt/pull/42175 +.. _`#42176`: https://github.com/saltstack/salt/pull/42176 +.. _`#42179`: https://github.com/saltstack/salt/pull/42179 +.. _`#42180`: https://github.com/saltstack/salt/pull/42180 +.. _`#42181`: https://github.com/saltstack/salt/pull/42181 +.. _`#42182`: https://github.com/saltstack/salt/pull/42182 +.. _`#42186`: https://github.com/saltstack/salt/pull/42186 +.. _`#42194`: https://github.com/saltstack/salt/issues/42194 +.. _`#42198`: https://github.com/saltstack/salt/issues/42198 +.. _`#42200`: https://github.com/saltstack/salt/pull/42200 +.. _`#42206`: https://github.com/saltstack/salt/pull/42206 +.. _`#42210`: https://github.com/saltstack/salt/pull/42210 +.. _`#42211`: https://github.com/saltstack/salt/pull/42211 +.. _`#42215`: https://github.com/saltstack/salt/pull/42215 +.. _`#42224`: https://github.com/saltstack/salt/pull/42224 +.. _`#42232`: https://github.com/saltstack/salt/issues/42232 +.. _`#42235`: https://github.com/saltstack/salt/pull/42235 +.. _`#42236`: https://github.com/saltstack/salt/pull/42236 +.. _`#42240`: https://github.com/saltstack/salt/issues/42240 +.. _`#42251`: https://github.com/saltstack/salt/pull/42251 +.. _`#42252`: https://github.com/saltstack/salt/pull/42252 +.. _`#42253`: https://github.com/saltstack/salt/pull/42253 +.. _`#42255`: https://github.com/saltstack/salt/pull/42255 +.. _`#42257`: https://github.com/saltstack/salt/pull/42257 +.. _`#42258`: https://github.com/saltstack/salt/pull/42258 +.. _`#42261`: https://github.com/saltstack/salt/pull/42261 +.. _`#42262`: https://github.com/saltstack/salt/pull/42262 +.. _`#42264`: https://github.com/saltstack/salt/pull/42264 +.. _`#42265`: https://github.com/saltstack/salt/pull/42265 +.. _`#42266`: https://github.com/saltstack/salt/pull/42266 +.. _`#42267`: https://github.com/saltstack/salt/issues/42267 +.. _`#42269`: https://github.com/saltstack/salt/pull/42269 +.. _`#42270`: https://github.com/saltstack/salt/issues/42270 +.. _`#42275`: https://github.com/saltstack/salt/pull/42275 +.. _`#42277`: https://github.com/saltstack/salt/pull/42277 +.. _`#42279`: https://github.com/saltstack/salt/issues/42279 +.. _`#42282`: https://github.com/saltstack/salt/pull/42282 +.. _`#42289`: https://github.com/saltstack/salt/pull/42289 +.. _`#42290`: https://github.com/saltstack/salt/pull/42290 +.. _`#42291`: https://github.com/saltstack/salt/pull/42291 +.. _`#42295`: https://github.com/saltstack/salt/issues/42295 +.. _`#42308`: https://github.com/saltstack/salt/pull/42308 +.. _`#42309`: https://github.com/saltstack/salt/pull/42309 +.. _`#42314`: https://github.com/saltstack/salt/pull/42314 +.. _`#42319`: https://github.com/saltstack/salt/pull/42319 +.. _`#42327`: https://github.com/saltstack/salt/pull/42327 +.. _`#42329`: https://github.com/saltstack/salt/issues/42329 +.. _`#42333`: https://github.com/saltstack/salt/issues/42333 +.. _`#42339`: https://github.com/saltstack/salt/pull/42339 +.. _`#42340`: https://github.com/saltstack/salt/pull/42340 +.. _`#42347`: https://github.com/saltstack/salt/pull/42347 +.. _`#42350`: https://github.com/saltstack/salt/pull/42350 +.. _`#42352`: https://github.com/saltstack/salt/pull/42352 +.. _`#42353`: https://github.com/saltstack/salt/pull/42353 +.. _`#42356`: https://github.com/saltstack/salt/pull/42356 +.. _`#42357`: https://github.com/saltstack/salt/issues/42357 +.. _`#42359`: https://github.com/saltstack/salt/pull/42359 +.. _`#42360`: https://github.com/saltstack/salt/pull/42360 +.. _`#42361`: https://github.com/saltstack/salt/pull/42361 +.. _`#42363`: https://github.com/saltstack/salt/pull/42363 +.. _`#42364`: https://github.com/saltstack/salt/pull/42364 +.. _`#42366`: https://github.com/saltstack/salt/pull/42366 +.. _`#42368`: https://github.com/saltstack/salt/pull/42368 +.. _`#42370`: https://github.com/saltstack/salt/pull/42370 +.. _`#42371`: https://github.com/saltstack/salt/issues/42371 +.. _`#42373`: https://github.com/saltstack/salt/pull/42373 +.. _`#42374`: https://github.com/saltstack/salt/issues/42374 +.. _`#42375`: https://github.com/saltstack/salt/issues/42375 +.. _`#42381`: https://github.com/saltstack/salt/issues/42381 +.. _`#42387`: https://github.com/saltstack/salt/pull/42387 +.. _`#42388`: https://github.com/saltstack/salt/pull/42388 +.. _`#42399`: https://github.com/saltstack/salt/pull/42399 +.. _`#42400`: https://github.com/saltstack/salt/issues/42400 +.. _`#42403`: https://github.com/saltstack/salt/issues/42403 +.. _`#42404`: https://github.com/saltstack/salt/issues/42404 +.. _`#42405`: https://github.com/saltstack/salt/issues/42405 +.. _`#42408`: https://github.com/saltstack/salt/pull/42408 +.. _`#42409`: https://github.com/saltstack/salt/pull/42409 +.. _`#42411`: https://github.com/saltstack/salt/pull/42411 +.. _`#42413`: https://github.com/saltstack/salt/issues/42413 +.. _`#42414`: https://github.com/saltstack/salt/pull/42414 +.. _`#42417`: https://github.com/saltstack/salt/issues/42417 +.. _`#42421`: https://github.com/saltstack/salt/issues/42421 +.. _`#42424`: https://github.com/saltstack/salt/pull/42424 +.. _`#42425`: https://github.com/saltstack/salt/pull/42425 +.. _`#42427`: https://github.com/saltstack/salt/issues/42427 +.. _`#42433`: https://github.com/saltstack/salt/pull/42433 +.. _`#42435`: https://github.com/saltstack/salt/pull/42435 +.. _`#42436`: https://github.com/saltstack/salt/pull/42436 +.. _`#42443`: https://github.com/saltstack/salt/pull/42443 +.. _`#42444`: https://github.com/saltstack/salt/pull/42444 +.. _`#42452`: https://github.com/saltstack/salt/pull/42452 +.. _`#42453`: https://github.com/saltstack/salt/pull/42453 +.. _`#42454`: https://github.com/saltstack/salt/pull/42454 +.. _`#42456`: https://github.com/saltstack/salt/issues/42456 +.. _`#42459`: https://github.com/saltstack/salt/issues/42459 +.. _`#42461`: https://github.com/saltstack/salt/pull/42461 +.. _`#42464`: https://github.com/saltstack/salt/pull/42464 +.. _`#42465`: https://github.com/saltstack/salt/pull/42465 +.. _`#42474`: https://github.com/saltstack/salt/pull/42474 +.. _`#42477`: https://github.com/saltstack/salt/issues/42477 +.. _`#42479`: https://github.com/saltstack/salt/pull/42479 +.. _`#42481`: https://github.com/saltstack/salt/pull/42481 +.. _`#42484`: https://github.com/saltstack/salt/pull/42484 +.. _`#42502`: https://github.com/saltstack/salt/pull/42502 +.. _`#42505`: https://github.com/saltstack/salt/issues/42505 +.. _`#42506`: https://github.com/saltstack/salt/pull/42506 +.. _`#42509`: https://github.com/saltstack/salt/pull/42509 +.. _`#42514`: https://github.com/saltstack/salt/issues/42514 +.. _`#42515`: https://github.com/saltstack/salt/pull/42515 +.. _`#42516`: https://github.com/saltstack/salt/pull/42516 +.. _`#42521`: https://github.com/saltstack/salt/issues/42521 +.. _`#42523`: https://github.com/saltstack/salt/pull/42523 +.. _`#42524`: https://github.com/saltstack/salt/pull/42524 +.. _`#42527`: https://github.com/saltstack/salt/pull/42527 +.. _`#42528`: https://github.com/saltstack/salt/pull/42528 +.. _`#42529`: https://github.com/saltstack/salt/pull/42529 +.. _`#42534`: https://github.com/saltstack/salt/pull/42534 +.. _`#42538`: https://github.com/saltstack/salt/issues/42538 +.. _`#42541`: https://github.com/saltstack/salt/pull/42541 +.. _`#42545`: https://github.com/saltstack/salt/issues/42545 +.. _`#42547`: https://github.com/saltstack/salt/pull/42547 +.. _`#42551`: https://github.com/saltstack/salt/pull/42551 +.. _`#42552`: https://github.com/saltstack/salt/pull/42552 +.. _`#42555`: https://github.com/saltstack/salt/pull/42555 +.. _`#42557`: https://github.com/saltstack/salt/pull/42557 +.. _`#42567`: https://github.com/saltstack/salt/pull/42567 +.. _`#42571`: https://github.com/saltstack/salt/pull/42571 +.. _`#42573`: https://github.com/saltstack/salt/pull/42573 +.. _`#42574`: https://github.com/saltstack/salt/pull/42574 +.. _`#42575`: https://github.com/saltstack/salt/pull/42575 +.. _`#42577`: https://github.com/saltstack/salt/pull/42577 +.. _`#42586`: https://github.com/saltstack/salt/pull/42586 +.. _`#42588`: https://github.com/saltstack/salt/issues/42588 +.. _`#42589`: https://github.com/saltstack/salt/pull/42589 +.. _`#42600`: https://github.com/saltstack/salt/issues/42600 +.. _`#42601`: https://github.com/saltstack/salt/pull/42601 +.. _`#42602`: https://github.com/saltstack/salt/pull/42602 +.. _`#42603`: https://github.com/saltstack/salt/pull/42603 +.. _`#42611`: https://github.com/saltstack/salt/issues/42611 +.. _`#42612`: https://github.com/saltstack/salt/pull/42612 +.. _`#42616`: https://github.com/saltstack/salt/pull/42616 +.. _`#42618`: https://github.com/saltstack/salt/pull/42618 +.. _`#42619`: https://github.com/saltstack/salt/pull/42619 +.. _`#42621`: https://github.com/saltstack/salt/pull/42621 +.. _`#42623`: https://github.com/saltstack/salt/pull/42623 +.. _`#42625`: https://github.com/saltstack/salt/pull/42625 +.. _`#42627`: https://github.com/saltstack/salt/issues/42627 +.. _`#42629`: https://github.com/saltstack/salt/pull/42629 +.. _`#42639`: https://github.com/saltstack/salt/issues/42639 +.. _`#42642`: https://github.com/saltstack/salt/issues/42642 +.. _`#42644`: https://github.com/saltstack/salt/issues/42644 +.. _`#42646`: https://github.com/saltstack/salt/issues/42646 +.. _`#42649`: https://github.com/saltstack/salt/issues/42649 +.. _`#42651`: https://github.com/saltstack/salt/pull/42651 +.. _`#42654`: https://github.com/saltstack/salt/pull/42654 +.. _`#42655`: https://github.com/saltstack/salt/pull/42655 +.. _`#42657`: https://github.com/saltstack/salt/pull/42657 +.. _`#42663`: https://github.com/saltstack/salt/pull/42663 +.. _`#42668`: https://github.com/saltstack/salt/issues/42668 +.. _`#42669`: https://github.com/saltstack/salt/pull/42669 +.. _`#42670`: https://github.com/saltstack/salt/pull/42670 +.. _`#42678`: https://github.com/saltstack/salt/pull/42678 +.. _`#42679`: https://github.com/saltstack/salt/pull/42679 +.. _`#42683`: https://github.com/saltstack/salt/issues/42683 +.. _`#42686`: https://github.com/saltstack/salt/issues/42686 +.. _`#42688`: https://github.com/saltstack/salt/issues/42688 +.. _`#42689`: https://github.com/saltstack/salt/pull/42689 +.. _`#42690`: https://github.com/saltstack/salt/issues/42690 +.. _`#42693`: https://github.com/saltstack/salt/pull/42693 +.. _`#42694`: https://github.com/saltstack/salt/pull/42694 +.. _`#42697`: https://github.com/saltstack/salt/issues/42697 +.. _`#42704`: https://github.com/saltstack/salt/pull/42704 +.. _`#42705`: https://github.com/saltstack/salt/issues/42705 +.. _`#42707`: https://github.com/saltstack/salt/issues/42707 +.. _`#42708`: https://github.com/saltstack/salt/pull/42708 +.. _`#42709`: https://github.com/saltstack/salt/pull/42709 +.. _`#42710`: https://github.com/saltstack/salt/pull/42710 +.. _`#42712`: https://github.com/saltstack/salt/pull/42712 +.. _`#42714`: https://github.com/saltstack/salt/pull/42714 +.. _`#42721`: https://github.com/saltstack/salt/pull/42721 +.. _`#42731`: https://github.com/saltstack/salt/issues/42731 +.. _`#42741`: https://github.com/saltstack/salt/issues/42741 +.. _`#42743`: https://github.com/saltstack/salt/pull/42743 +.. _`#42744`: https://github.com/saltstack/salt/pull/42744 +.. _`#42745`: https://github.com/saltstack/salt/pull/42745 +.. _`#42747`: https://github.com/saltstack/salt/issues/42747 +.. _`#42748`: https://github.com/saltstack/salt/pull/42748 +.. _`#42753`: https://github.com/saltstack/salt/issues/42753 +.. _`#42760`: https://github.com/saltstack/salt/pull/42760 +.. _`#42764`: https://github.com/saltstack/salt/pull/42764 +.. _`#42768`: https://github.com/saltstack/salt/pull/42768 +.. _`#42769`: https://github.com/saltstack/salt/pull/42769 +.. _`#42770`: https://github.com/saltstack/salt/pull/42770 +.. _`#42774`: https://github.com/saltstack/salt/issues/42774 +.. _`#42778`: https://github.com/saltstack/salt/pull/42778 +.. _`#42782`: https://github.com/saltstack/salt/pull/42782 +.. _`#42783`: https://github.com/saltstack/salt/pull/42783 +.. _`#42784`: https://github.com/saltstack/salt/pull/42784 +.. _`#42786`: https://github.com/saltstack/salt/pull/42786 +.. _`#42788`: https://github.com/saltstack/salt/pull/42788 +.. _`#42794`: https://github.com/saltstack/salt/pull/42794 +.. _`#42795`: https://github.com/saltstack/salt/pull/42795 +.. _`#42798`: https://github.com/saltstack/salt/pull/42798 +.. _`#42800`: https://github.com/saltstack/salt/pull/42800 +.. _`#42803`: https://github.com/saltstack/salt/issues/42803 +.. _`#42804`: https://github.com/saltstack/salt/pull/42804 +.. _`#42805`: https://github.com/saltstack/salt/pull/42805 +.. _`#42806`: https://github.com/saltstack/salt/pull/42806 +.. _`#42807`: https://github.com/saltstack/salt/pull/42807 +.. _`#42808`: https://github.com/saltstack/salt/pull/42808 +.. _`#42810`: https://github.com/saltstack/salt/pull/42810 +.. _`#42812`: https://github.com/saltstack/salt/pull/42812 +.. _`#42818`: https://github.com/saltstack/salt/issues/42818 +.. _`#42826`: https://github.com/saltstack/salt/pull/42826 +.. _`#42829`: https://github.com/saltstack/salt/pull/42829 +.. _`#42835`: https://github.com/saltstack/salt/pull/42835 +.. _`#42836`: https://github.com/saltstack/salt/pull/42836 +.. _`#42838`: https://github.com/saltstack/salt/pull/42838 +.. _`#42841`: https://github.com/saltstack/salt/pull/42841 +.. _`#42842`: https://github.com/saltstack/salt/issues/42842 +.. _`#42843`: https://github.com/saltstack/salt/issues/42843 +.. _`#42845`: https://github.com/saltstack/salt/pull/42845 +.. _`#42848`: https://github.com/saltstack/salt/pull/42848 +.. _`#42851`: https://github.com/saltstack/salt/pull/42851 +.. _`#42855`: https://github.com/saltstack/salt/pull/42855 +.. _`#42856`: https://github.com/saltstack/salt/pull/42856 +.. _`#42857`: https://github.com/saltstack/salt/pull/42857 +.. _`#42859`: https://github.com/saltstack/salt/pull/42859 +.. _`#42860`: https://github.com/saltstack/salt/pull/42860 +.. _`#42861`: https://github.com/saltstack/salt/pull/42861 +.. _`#42864`: https://github.com/saltstack/salt/pull/42864 +.. _`#42866`: https://github.com/saltstack/salt/pull/42866 +.. _`#42868`: https://github.com/saltstack/salt/pull/42868 +.. _`#42869`: https://github.com/saltstack/salt/issues/42869 +.. _`#42870`: https://github.com/saltstack/salt/issues/42870 +.. _`#42871`: https://github.com/saltstack/salt/pull/42871 +.. _`#42873`: https://github.com/saltstack/salt/issues/42873 +.. _`#42877`: https://github.com/saltstack/salt/pull/42877 +.. _`#42881`: https://github.com/saltstack/salt/pull/42881 +.. _`#42882`: https://github.com/saltstack/salt/pull/42882 +.. _`#42883`: https://github.com/saltstack/salt/pull/42883 +.. _`#42884`: https://github.com/saltstack/salt/pull/42884 +.. _`#42885`: https://github.com/saltstack/salt/pull/42885 +.. _`#42886`: https://github.com/saltstack/salt/pull/42886 +.. _`#42887`: https://github.com/saltstack/salt/pull/42887 +.. _`#42889`: https://github.com/saltstack/salt/pull/42889 +.. _`#42890`: https://github.com/saltstack/salt/pull/42890 +.. _`#42898`: https://github.com/saltstack/salt/pull/42898 +.. _`#42911`: https://github.com/saltstack/salt/pull/42911 +.. _`#42913`: https://github.com/saltstack/salt/pull/42913 +.. _`#42918`: https://github.com/saltstack/salt/pull/42918 +.. _`#42919`: https://github.com/saltstack/salt/pull/42919 +.. _`#42920`: https://github.com/saltstack/salt/pull/42920 +.. _`#42925`: https://github.com/saltstack/salt/pull/42925 +.. _`#42933`: https://github.com/saltstack/salt/pull/42933 +.. _`#42936`: https://github.com/saltstack/salt/issues/42936 +.. _`#42940`: https://github.com/saltstack/salt/pull/42940 +.. _`#42941`: https://github.com/saltstack/salt/issues/42941 +.. _`#42942`: https://github.com/saltstack/salt/pull/42942 +.. _`#42943`: https://github.com/saltstack/salt/issues/42943 +.. _`#42944`: https://github.com/saltstack/salt/pull/42944 +.. _`#42945`: https://github.com/saltstack/salt/pull/42945 +.. _`#42946`: https://github.com/saltstack/salt/pull/42946 +.. _`#42948`: https://github.com/saltstack/salt/pull/42948 +.. _`#42949`: https://github.com/saltstack/salt/pull/42949 +.. _`#42950`: https://github.com/saltstack/salt/pull/42950 +.. _`#42951`: https://github.com/saltstack/salt/pull/42951 +.. _`#42952`: https://github.com/saltstack/salt/pull/42952 +.. _`#42953`: https://github.com/saltstack/salt/pull/42953 +.. _`#42954`: https://github.com/saltstack/salt/pull/42954 +.. _`#42958`: https://github.com/saltstack/salt/pull/42958 +.. _`#42959`: https://github.com/saltstack/salt/pull/42959 +.. _`#42962`: https://github.com/saltstack/salt/pull/42962 +.. _`#42963`: https://github.com/saltstack/salt/pull/42963 +.. _`#42964`: https://github.com/saltstack/salt/pull/42964 +.. _`#42967`: https://github.com/saltstack/salt/pull/42967 +.. _`#42968`: https://github.com/saltstack/salt/pull/42968 +.. _`#42969`: https://github.com/saltstack/salt/pull/42969 +.. _`#42985`: https://github.com/saltstack/salt/pull/42985 +.. _`#42986`: https://github.com/saltstack/salt/pull/42986 +.. _`#42988`: https://github.com/saltstack/salt/pull/42988 +.. _`#42989`: https://github.com/saltstack/salt/issues/42989 +.. _`#42992`: https://github.com/saltstack/salt/issues/42992 +.. _`#42993`: https://github.com/saltstack/salt/pull/42993 +.. _`#42995`: https://github.com/saltstack/salt/pull/42995 +.. _`#42996`: https://github.com/saltstack/salt/pull/42996 +.. _`#42997`: https://github.com/saltstack/salt/pull/42997 +.. _`#42999`: https://github.com/saltstack/salt/pull/42999 +.. _`#43002`: https://github.com/saltstack/salt/pull/43002 +.. _`#43006`: https://github.com/saltstack/salt/pull/43006 +.. _`#43008`: https://github.com/saltstack/salt/issues/43008 +.. _`#43009`: https://github.com/saltstack/salt/pull/43009 +.. _`#43010`: https://github.com/saltstack/salt/pull/43010 +.. _`#43014`: https://github.com/saltstack/salt/pull/43014 +.. _`#43016`: https://github.com/saltstack/salt/pull/43016 +.. _`#43019`: https://github.com/saltstack/salt/pull/43019 +.. _`#43020`: https://github.com/saltstack/salt/pull/43020 +.. _`#43021`: https://github.com/saltstack/salt/pull/43021 +.. _`#43023`: https://github.com/saltstack/salt/pull/43023 +.. _`#43024`: https://github.com/saltstack/salt/pull/43024 +.. _`#43026`: https://github.com/saltstack/salt/pull/43026 +.. _`#43027`: https://github.com/saltstack/salt/pull/43027 +.. _`#43029`: https://github.com/saltstack/salt/pull/43029 +.. _`#43030`: https://github.com/saltstack/salt/pull/43030 +.. _`#43031`: https://github.com/saltstack/salt/pull/43031 +.. _`#43032`: https://github.com/saltstack/salt/pull/43032 +.. _`#43033`: https://github.com/saltstack/salt/pull/43033 +.. _`#43034`: https://github.com/saltstack/salt/pull/43034 +.. _`#43035`: https://github.com/saltstack/salt/pull/43035 +.. _`#43036`: https://github.com/saltstack/salt/issues/43036 +.. _`#43037`: https://github.com/saltstack/salt/pull/43037 +.. _`#43038`: https://github.com/saltstack/salt/pull/43038 +.. _`#43039`: https://github.com/saltstack/salt/pull/43039 +.. _`#43040`: https://github.com/saltstack/salt/issues/43040 +.. _`#43041`: https://github.com/saltstack/salt/pull/43041 +.. _`#43043`: https://github.com/saltstack/salt/issues/43043 +.. _`#43048`: https://github.com/saltstack/salt/pull/43048 +.. _`#43051`: https://github.com/saltstack/salt/pull/43051 +.. _`#43054`: https://github.com/saltstack/salt/pull/43054 +.. _`#43056`: https://github.com/saltstack/salt/pull/43056 +.. _`#43058`: https://github.com/saltstack/salt/pull/43058 +.. _`#43060`: https://github.com/saltstack/salt/pull/43060 +.. _`#43061`: https://github.com/saltstack/salt/pull/43061 +.. _`#43064`: https://github.com/saltstack/salt/pull/43064 +.. _`#43068`: https://github.com/saltstack/salt/pull/43068 +.. _`#43073`: https://github.com/saltstack/salt/pull/43073 +.. _`#43077`: https://github.com/saltstack/salt/issues/43077 +.. _`#43085`: https://github.com/saltstack/salt/issues/43085 +.. _`#43087`: https://github.com/saltstack/salt/pull/43087 +.. _`#43088`: https://github.com/saltstack/salt/pull/43088 +.. _`#43091`: https://github.com/saltstack/salt/pull/43091 +.. _`#43092`: https://github.com/saltstack/salt/pull/43092 +.. _`#43093`: https://github.com/saltstack/salt/pull/43093 +.. _`#43097`: https://github.com/saltstack/salt/pull/43097 +.. _`#43100`: https://github.com/saltstack/salt/pull/43100 +.. _`#43101`: https://github.com/saltstack/salt/issues/43101 +.. _`#43103`: https://github.com/saltstack/salt/pull/43103 +.. _`#43107`: https://github.com/saltstack/salt/pull/43107 +.. _`#43108`: https://github.com/saltstack/salt/pull/43108 +.. _`#43110`: https://github.com/saltstack/salt/issues/43110 +.. _`#43115`: https://github.com/saltstack/salt/pull/43115 +.. _`#43116`: https://github.com/saltstack/salt/pull/43116 +.. _`#43123`: https://github.com/saltstack/salt/pull/43123 +.. _`#43142`: https://github.com/saltstack/salt/pull/43142 +.. _`#43143`: https://github.com/saltstack/salt/issues/43143 +.. _`#43146`: https://github.com/saltstack/salt/pull/43146 +.. _`#43151`: https://github.com/saltstack/salt/pull/43151 +.. _`#43155`: https://github.com/saltstack/salt/pull/43155 +.. _`#43156`: https://github.com/saltstack/salt/pull/43156 +.. _`#43162`: https://github.com/saltstack/salt/issues/43162 +.. _`#43165`: https://github.com/saltstack/salt/pull/43165 +.. _`#43166`: https://github.com/saltstack/salt/pull/43166 +.. _`#43168`: https://github.com/saltstack/salt/pull/43168 +.. _`#43170`: https://github.com/saltstack/salt/pull/43170 +.. _`#43171`: https://github.com/saltstack/salt/pull/43171 +.. _`#43172`: https://github.com/saltstack/salt/pull/43172 +.. _`#43173`: https://github.com/saltstack/salt/pull/43173 +.. _`#43178`: https://github.com/saltstack/salt/pull/43178 +.. _`#43179`: https://github.com/saltstack/salt/pull/43179 +.. _`#43184`: https://github.com/saltstack/salt/pull/43184 +.. _`#43193`: https://github.com/saltstack/salt/pull/43193 +.. _`#43196`: https://github.com/saltstack/salt/pull/43196 +.. _`#43198`: https://github.com/saltstack/salt/issues/43198 +.. _`#43199`: https://github.com/saltstack/salt/pull/43199 +.. _`#43201`: https://github.com/saltstack/salt/pull/43201 +.. _`#43202`: https://github.com/saltstack/salt/pull/43202 +.. _`#43217`: https://github.com/saltstack/salt/pull/43217 +.. _`#43226`: https://github.com/saltstack/salt/pull/43226 +.. _`#43227`: https://github.com/saltstack/salt/pull/43227 +.. _`#43228`: https://github.com/saltstack/salt/pull/43228 +.. _`#43229`: https://github.com/saltstack/salt/pull/43229 +.. _`#43241`: https://github.com/saltstack/salt/issues/43241 +.. _`#43251`: https://github.com/saltstack/salt/pull/43251 +.. _`#43254`: https://github.com/saltstack/salt/pull/43254 +.. _`#43255`: https://github.com/saltstack/salt/pull/43255 +.. _`#43256`: https://github.com/saltstack/salt/pull/43256 +.. _`#43259`: https://github.com/saltstack/salt/issues/43259 +.. _`#43266`: https://github.com/saltstack/salt/pull/43266 +.. _`#43283`: https://github.com/saltstack/salt/pull/43283 +.. _`#43315`: https://github.com/saltstack/salt/pull/43315 +.. _`#43330`: https://github.com/saltstack/salt/pull/43330 +.. _`#43333`: https://github.com/saltstack/salt/pull/43333 +.. _`#43377`: https://github.com/saltstack/salt/pull/43377 +.. _`#43421`: https://github.com/saltstack/salt/pull/43421 +.. _`#43440`: https://github.com/saltstack/salt/pull/43440 +.. _`#43447`: https://github.com/saltstack/salt/issues/43447 +.. _`#43509`: https://github.com/saltstack/salt/pull/43509 +.. _`#43526`: https://github.com/saltstack/salt/pull/43526 +.. _`#43551`: https://github.com/saltstack/salt/pull/43551 +.. _`#43585`: https://github.com/saltstack/salt/pull/43585 +.. _`#43586`: https://github.com/saltstack/salt/pull/43586 +.. _`#43756`: https://github.com/saltstack/salt/pull/43756 +.. _`#43847`: https://github.com/saltstack/salt/pull/43847 +.. _`#43868`: https://github.com/saltstack/salt/pull/43868 +.. _`#475`: https://github.com/saltstack/salt/issues/475 +.. _`#480`: https://github.com/saltstack/salt/issues/480 +.. _`#495`: https://github.com/saltstack/salt/issues/495 +.. _`bp-37424`: https://github.com/saltstack/salt/pull/37424 +.. _`bp-39366`: https://github.com/saltstack/salt/pull/39366 +.. _`bp-41543`: https://github.com/saltstack/salt/pull/41543 +.. _`bp-41690`: https://github.com/saltstack/salt/pull/41690 +.. _`bp-42067`: https://github.com/saltstack/salt/pull/42067 +.. _`bp-42097`: https://github.com/saltstack/salt/pull/42097 +.. _`bp-42098`: https://github.com/saltstack/salt/pull/42098 +.. _`bp-42109`: https://github.com/saltstack/salt/pull/42109 +.. _`bp-42174`: https://github.com/saltstack/salt/pull/42174 +.. _`bp-42224`: https://github.com/saltstack/salt/pull/42224 +.. _`bp-42433`: https://github.com/saltstack/salt/pull/42433 +.. _`bp-42547`: https://github.com/saltstack/salt/pull/42547 +.. _`bp-42552`: https://github.com/saltstack/salt/pull/42552 +.. _`bp-42589`: https://github.com/saltstack/salt/pull/42589 +.. _`bp-42651`: https://github.com/saltstack/salt/pull/42651 +.. _`bp-42744`: https://github.com/saltstack/salt/pull/42744 +.. _`bp-42760`: https://github.com/saltstack/salt/pull/42760 +.. _`bp-42784`: https://github.com/saltstack/salt/pull/42784 +.. _`bp-42848`: https://github.com/saltstack/salt/pull/42848 +.. _`bp-42871`: https://github.com/saltstack/salt/pull/42871 +.. _`bp-42883`: https://github.com/saltstack/salt/pull/42883 +.. _`bp-42988`: https://github.com/saltstack/salt/pull/42988 +.. _`bp-43002`: https://github.com/saltstack/salt/pull/43002 +.. _`bp-43020`: https://github.com/saltstack/salt/pull/43020 +.. _`bp-43031`: https://github.com/saltstack/salt/pull/43031 +.. _`bp-43041`: https://github.com/saltstack/salt/pull/43041 +.. _`bp-43068`: https://github.com/saltstack/salt/pull/43068 +.. _`bp-43116`: https://github.com/saltstack/salt/pull/43116 +.. _`bp-43193`: https://github.com/saltstack/salt/pull/43193 +.. _`bp-43283`: https://github.com/saltstack/salt/pull/43283 +.. _`bp-43330`: https://github.com/saltstack/salt/pull/43330 +.. _`bp-43333`: https://github.com/saltstack/salt/pull/43333 +.. _`bp-43421`: https://github.com/saltstack/salt/pull/43421 +.. _`bp-43526`: https://github.com/saltstack/salt/pull/43526 +.. _`fix-38839`: https://github.com/saltstack/salt/issues/38839 +.. _`fix-41116`: https://github.com/saltstack/salt/issues/41116 +.. _`fix-41721`: https://github.com/saltstack/salt/issues/41721 +.. _`fix-41885`: https://github.com/saltstack/salt/issues/41885 +.. _`fix-42115`: https://github.com/saltstack/salt/issues/42115 +.. _`fix-42151`: https://github.com/saltstack/salt/issues/42151 +.. _`fix-42152`: https://github.com/saltstack/salt/issues/42152 +.. _`fix-42166`: https://github.com/saltstack/salt/issues/42166 +.. _`fix-42267`: https://github.com/saltstack/salt/issues/42267 +.. _`fix-42375`: https://github.com/saltstack/salt/issues/42375 +.. _`fix-42381`: https://github.com/saltstack/salt/issues/42381 +.. _`fix-42405`: https://github.com/saltstack/salt/issues/42405 +.. _`fix-42417`: https://github.com/saltstack/salt/issues/42417 +.. _`fix-42639`: https://github.com/saltstack/salt/issues/42639 +.. _`fix-42683`: https://github.com/saltstack/salt/issues/42683 +.. _`fix-42697`: https://github.com/saltstack/salt/issues/42697 +.. _`fix-42870`: https://github.com/saltstack/salt/issues/42870 From 04a23f9f9ba6b0323c29ff80b0a7193256d75660 Mon Sep 17 00:00:00 2001 From: Alex moore Date: Tue, 3 Oct 2017 16:20:04 -0600 Subject: [PATCH 018/184] Add container_ref to specify device creation and network dvs maping --- salt/cloud/clouds/vmware.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/cloud/clouds/vmware.py b/salt/cloud/clouds/vmware.py index 98ab762230..eb823047eb 100644 --- a/salt/cloud/clouds/vmware.py +++ b/salt/cloud/clouds/vmware.py @@ -704,7 +704,7 @@ def _manage_devices(devices, vm=None, container_ref=None, new_vm_name=None): network_name = devices['network'][device.deviceInfo.label]['name'] adapter_type = devices['network'][device.deviceInfo.label]['adapter_type'] if 'adapter_type' in devices['network'][device.deviceInfo.label] else '' switch_type = devices['network'][device.deviceInfo.label]['switch_type'] if 'switch_type' in devices['network'][device.deviceInfo.label] else '' - network_spec = _edit_existing_network_adapter(device, network_name, adapter_type, switch_type) + network_spec = _edit_existing_network_adapter(device, network_name, adapter_type, switch_type, container_ref) adapter_mapping = _set_network_adapter_mapping(devices['network'][device.deviceInfo.label]) device_specs.append(network_spec) nics_map.append(adapter_mapping) @@ -2577,7 +2577,7 @@ def create(vm_): config_spec.memoryMB = memory_mb if devices: - specs = _manage_devices(devices, vm=object_ref, new_vm_name=vm_name) + specs = _manage_devices(devices, vm=object_ref, container_ref=container_ref, new_vm_name=vm_name) config_spec.deviceChange = specs['device_specs'] if extra_config: From bd04b29e9ab65d621e81f20f8a89d05db942f0cb Mon Sep 17 00:00:00 2001 From: Nasenbaer Date: Wed, 4 Oct 2017 12:42:34 +0200 Subject: [PATCH 019/184] Encode tags as utf-8, retry policy readout --- salt/modules/boto_asg.py | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/salt/modules/boto_asg.py b/salt/modules/boto_asg.py index 294c00b8f9..ce8bfc8262 100644 --- a/salt/modules/boto_asg.py +++ b/salt/modules/boto_asg.py @@ -51,6 +51,7 @@ import datetime import logging import json import sys +import time import email.mime.multipart log = logging.getLogger(__name__) @@ -677,11 +678,22 @@ def get_scaling_policy_arn(as_group, scaling_policy_name, region=None, salt '*' boto_asg.get_scaling_policy_arn mygroup mypolicy ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - policies = conn.get_all_policies(as_group=as_group) - for policy in policies: - if policy.name == scaling_policy_name: - return policy.policy_arn - log.error('Could not convert: {0}'.format(as_group)) + retries = 30 + while retries > 0: + retries -= 1 + try: + policies = conn.get_all_policies(as_group=as_group) + for policy in policies: + if policy.name == scaling_policy_name: + return policy.policy_arn + log.error('Could not convert: {0}'.format(as_group)) + return None + except boto.exception.BotoServerError as e: + if e.error_code != 'Throttling': + raise + time.sleep(5) + + log.error('Maximum number of retries exceeded') return None @@ -763,11 +775,18 @@ def get_instances(name, lifecycle_state="InService", health_status="Healthy", # get full instance info, so that we can return the attribute instances = ec2_conn.get_only_instances(instance_ids=instance_ids) if attributes: - return [[getattr(instance, attr).encode("ascii") for attr in attributes] for instance in instances] + return [[_convert_attribute(instance, attr) for attr in attributes] for instance in instances] else: # properly handle case when not all instances have the requested attribute - return [getattr(instance, attribute).encode("ascii") for instance in instances if getattr(instance, attribute)] - return [getattr(instance, attribute).encode("ascii") for instance in instances] + return [_convert_attribute(instance, attribute) for instance in instances if getattr(instance, attribute)] + + +def _convert_attribute(instance, attribute): + if attribute == "tags": + tags = dict(getattr(instance, attribute)) + return {key.encode("utf-8"): value.encode("utf-8") for key, value in six.iteritems(tags)} + + return getattr(instance, attribute).encode("ascii") def enter_standby(name, instance_ids, should_decrement_desired_capacity=False, From 4c1b50fb5af33d1815ae0e75e8ea51b1ea8491a9 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 4 Oct 2017 09:14:20 -0600 Subject: [PATCH 020/184] Add space after comma --- salt/returners/pgjsonb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/returners/pgjsonb.py b/salt/returners/pgjsonb.py index 63afed1cda..30d8be02d6 100644 --- a/salt/returners/pgjsonb.py +++ b/salt/returners/pgjsonb.py @@ -415,7 +415,7 @@ def get_jids_filter(count, filter_find_job=True): data = cur.fetchall() ret = [] for jid, load in data: - ret.append(salt.utils.jid.format_jid_instance_ext(jid,load)) + ret.append(salt.utils.jid.format_jid_instance_ext(jid, load)) return ret From be91fbb7b78a7ad0213d96e1a05790ba0d96e662 Mon Sep 17 00:00:00 2001 From: Nasenbaer Date: Fri, 6 Oct 2017 09:05:42 +0200 Subject: [PATCH 021/184] Debug log added when throttled by API --- salt/modules/boto_asg.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/modules/boto_asg.py b/salt/modules/boto_asg.py index ce8bfc8262..2a6d8f4122 100644 --- a/salt/modules/boto_asg.py +++ b/salt/modules/boto_asg.py @@ -691,6 +691,7 @@ def get_scaling_policy_arn(as_group, scaling_policy_name, region=None, except boto.exception.BotoServerError as e: if e.error_code != 'Throttling': raise + log.debug('Throttled by API, will retry in 5 seconds') time.sleep(5) log.error('Maximum number of retries exceeded') From a63e6ca963117fa43a33dae2caf425bf7d99fd80 Mon Sep 17 00:00:00 2001 From: Massimiliano Torromeo Date: Wed, 27 Sep 2017 11:14:40 +0200 Subject: [PATCH 022/184] Copy git ssh-id-wrapper to /tmp only if necessary (Fixes #10582, Fixes #19532) This adds a check that only copies the ssh wrapper to a temporary location if git is to be run by a specific user and at the same time the predefined wrapper file is not executable by "others" by verifying every path part for the others executable bit. By doing this the temp file is kept only as a last resort which should work with salt-ssh as per bug #19532, while avoiding a needless copy on /tmp which could be potentially mounted with noexec as per bug #10582. --- salt/modules/git.py | 52 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index 304276187b..d862961a23 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -9,6 +9,7 @@ import copy import logging import os import re +import stat # Import salt libs import salt.utils @@ -115,6 +116,22 @@ def _expand_path(cwd, user): return os.path.join(os.path.expanduser(to_expand), str(cwd)) +def _path_is_executable_others(path): + ''' + Check every part of path for executable permission + ''' + prevpath = None + while path and path != prevpath: + try: + if not os.stat(path).st_mode & stat.S_IXOTH: + return False + except OSError: + return False + prevpath = path + path, _ = os.path.split(path) + return True + + def _format_opts(opts): ''' Common code to inspect opts and split them if necessary @@ -214,11 +231,12 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, } # copy wrapper to area accessible by ``runas`` user - # currently no suppport in windows for wrapping git ssh + # currently no support in windows for wrapping git ssh ssh_id_wrapper = os.path.join( salt.utils.templates.TEMPLATE_DIRNAME, 'git/ssh-id-wrapper' ) + tmp_ssh_wrapper = None if salt.utils.is_windows(): for suffix in ('', ' (x86)'): ssh_exe = ( @@ -235,12 +253,14 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, # Use the windows batch file instead of the bourne shell script ssh_id_wrapper += '.bat' env['GIT_SSH'] = ssh_id_wrapper + elif not user or _path_is_executable_others(ssh_id_wrapper): + env['GIT_SSH'] = ssh_id_wrapper else: - tmp_file = salt.utils.files.mkstemp() - salt.utils.files.copyfile(ssh_id_wrapper, tmp_file) - os.chmod(tmp_file, 0o500) - os.chown(tmp_file, __salt__['file.user_to_uid'](user), -1) - env['GIT_SSH'] = tmp_file + tmp_ssh_wrapper = salt.utils.files.mkstemp() + salt.utils.files.copyfile(ssh_id_wrapper, tmp_ssh_wrapper) + os.chmod(tmp_ssh_wrapper, 0o500) + os.chown(tmp_ssh_wrapper, __salt__['file.user_to_uid'](user), -1) + env['GIT_SSH'] = tmp_ssh_wrapper if 'salt-call' not in _salt_cli \ and __salt__['ssh.key_is_encrypted'](id_file): @@ -270,13 +290,25 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, redirect_stderr=redirect_stderr, **kwargs) finally: - if not salt.utils.is_windows() and 'GIT_SSH' in env: - os.remove(env['GIT_SSH']) + # Cleanup the temporary ssh wrapper file + try: + __salt__['file.remove'](tmp_ssh_wrapper) + log.debug('Removed ssh wrapper file %s', tmp_ssh_wrapper) + except AttributeError: + # No wrapper was used + pass + except (SaltInvocationError, CommandExecutionError) as exc: + log.warning('Failed to remove ssh wrapper file %s: %s', tmp_ssh_wrapper, exc) # Cleanup the temporary identity file - if tmp_identity_file and os.path.exists(tmp_identity_file): - log.debug('Removing identity file {0}'.format(tmp_identity_file)) + try: __salt__['file.remove'](tmp_identity_file) + log.debug('Removed identity file %s', tmp_identity_file) + except AttributeError: + # No identify file was used + pass + except (SaltInvocationError, CommandExecutionError) as exc: + log.warning('Failed to remove identity file %s: %s', tmp_identity_file, exc) # If the command was successful, no need to try additional IDs if result['retcode'] == 0: From 00dbba571219f8aa8adbf0695ba6c43031983320 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 4 Oct 2017 17:31:37 -0600 Subject: [PATCH 023/184] Fix `unit.test_pillar` for Windows Use delete=True in NamedTemporaryFile command otherwise the handle to the file isn't released and the file can't be opened by the actual test. --- tests/unit/test_pillar.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/unit/test_pillar.py b/tests/unit/test_pillar.py index 2c385f8085..5076e70b44 100644 --- a/tests/unit/test_pillar.py +++ b/tests/unit/test_pillar.py @@ -192,7 +192,7 @@ class PillarTestCase(TestCase): def _setup_test_topfile_mocks(self, Matcher, get_file_client, nodegroup_order, glob_order): # Write a simple topfile and two pillar state files - self.top_file = tempfile.NamedTemporaryFile(dir=TMP) + self.top_file = tempfile.NamedTemporaryFile(dir=TMP, delete=False) s = ''' base: group: @@ -209,19 +209,19 @@ base: '''.format(nodegroup_order=nodegroup_order, glob_order=glob_order) self.top_file.write(salt.utils.to_bytes(s)) self.top_file.flush() - self.ssh_file = tempfile.NamedTemporaryFile(dir=TMP) + self.ssh_file = tempfile.NamedTemporaryFile(dir=TMP, delete=False) self.ssh_file.write(b''' ssh: foo ''') self.ssh_file.flush() - self.ssh_minion_file = tempfile.NamedTemporaryFile(dir=TMP) + self.ssh_minion_file = tempfile.NamedTemporaryFile(dir=TMP, delete=False) self.ssh_minion_file.write(b''' ssh: bar ''') self.ssh_minion_file.flush() - self.generic_file = tempfile.NamedTemporaryFile(dir=TMP) + self.generic_file = tempfile.NamedTemporaryFile(dir=TMP, delete=False) self.generic_file.write(b''' generic: key1: @@ -231,7 +231,7 @@ generic: sub_key1: [] ''') self.generic_file.flush() - self.generic_minion_file = tempfile.NamedTemporaryFile(dir=TMP) + self.generic_minion_file = tempfile.NamedTemporaryFile(dir=TMP, delete=False) self.generic_minion_file.write(b''' generic: key1: @@ -260,7 +260,7 @@ generic: client.get_state.side_effect = get_state def _setup_test_include_mocks(self, Matcher, get_file_client): - self.top_file = top_file = tempfile.NamedTemporaryFile(dir=TMP) + self.top_file = top_file = tempfile.NamedTemporaryFile(dir=TMP, delete=False) top_file.write(b''' base: '*': @@ -271,21 +271,21 @@ base: - test ''') top_file.flush() - self.init_sls = init_sls = tempfile.NamedTemporaryFile(dir=TMP) + self.init_sls = init_sls = tempfile.NamedTemporaryFile(dir=TMP, delete=False) init_sls.write(b''' include: - test.sub1 - test.sub2 ''') init_sls.flush() - self.sub1_sls = sub1_sls = tempfile.NamedTemporaryFile(dir=TMP) + self.sub1_sls = sub1_sls = tempfile.NamedTemporaryFile(dir=TMP, delete=False) sub1_sls.write(b''' p1: - value1_1 - value1_2 ''') sub1_sls.flush() - self.sub2_sls = sub2_sls = tempfile.NamedTemporaryFile(dir=TMP) + self.sub2_sls = sub2_sls = tempfile.NamedTemporaryFile(dir=TMP, delete=False) sub2_sls.write(b''' p1: - value1_3 From 44060dc9c1cc6c84f2e2b2ce11100e307fcf9d0d Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 25 Aug 2017 14:15:58 -0500 Subject: [PATCH 024/184] Do not allow IDs with null bytes in decoded payloads --- salt/crypt.py | 3 +++ salt/transport/tcp.py | 11 +++++++++++ salt/transport/zeromq.py | 11 +++++++++++ 3 files changed, 25 insertions(+) diff --git a/salt/crypt.py b/salt/crypt.py index f4f65940d5..ef4e7645d2 100644 --- a/salt/crypt.py +++ b/salt/crypt.py @@ -607,6 +607,9 @@ class AsyncAuth(object): raise tornado.gen.Return('retry') else: raise SaltClientError('Attempt to authenticate with the salt master failed with timeout error') + if not isinstance(payload, dict): + log.error('Sign-in attempt failed: %s', payload) + raise tornado.gen.Return(False) if 'load' in payload: if 'ret' in payload['load']: if not payload['load']['ret']: diff --git a/salt/transport/tcp.py b/salt/transport/tcp.py index a9001f03a5..f274240a1e 100644 --- a/salt/transport/tcp.py +++ b/salt/transport/tcp.py @@ -623,6 +623,17 @@ class TCPReqServerChannel(salt.transport.mixins.auth.AESReqServerMixin, salt.tra 'payload and load must be a dict', header=header)) raise tornado.gen.Return() + try: + id_ = payload['load'].get('id', '') + if '\0' in id_: + log.error('Payload contains an id with a null byte: %s', payload) + stream.send(self.serial.dumps('bad load: id contains a null byte')) + raise tornado.gen.Return() + except TypeError: + log.error('Payload contains non-string id: %s', payload) + stream.send(self.serial.dumps('bad load: id {0} is not a string'.format(id_))) + raise tornado.gen.Return() + # intercept the "_auth" commands, since the main daemon shouldn't know # anything about our key auth if payload['enc'] == 'clear' and payload.get('load', {}).get('cmd') == '_auth': diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py index 1bb3abebe1..2be6c829f3 100644 --- a/salt/transport/zeromq.py +++ b/salt/transport/zeromq.py @@ -596,6 +596,17 @@ class ZeroMQReqServerChannel(salt.transport.mixins.auth.AESReqServerMixin, salt. stream.send(self.serial.dumps('payload and load must be a dict')) raise tornado.gen.Return() + try: + id_ = payload['load'].get('id', '') + if '\0' in id_: + log.error('Payload contains an id with a null byte: %s', payload) + stream.send(self.serial.dumps('bad load: id contains a null byte')) + raise tornado.gen.Return() + except TypeError: + log.error('Payload contains non-string id: %s', payload) + stream.send(self.serial.dumps('bad load: id {0} is not a string'.format(id_))) + raise tornado.gen.Return() + # intercept the "_auth" commands, since the main daemon shouldn't know # anything about our key auth if payload['enc'] == 'clear' and payload.get('load', {}).get('cmd') == '_auth': From f7824e41f3105c7a7e6571d8fb66c8671ff94b74 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 23 Aug 2017 10:20:50 -0500 Subject: [PATCH 025/184] Don't allow path separators in minion ID --- salt/utils/verify.py | 15 ++++----------- tests/unit/utils/test_verify.py | 10 ++++++++++ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/salt/utils/verify.py b/salt/utils/verify.py index 6e69283f07..24b2a1d962 100644 --- a/salt/utils/verify.py +++ b/salt/utils/verify.py @@ -480,22 +480,15 @@ def clean_path(root, path, subdir=False): return '' -def clean_id(id_): - ''' - Returns if the passed id is clean. - ''' - if re.search(r'\.\.\{sep}'.format(sep=os.sep), id_): - return False - return True - - def valid_id(opts, id_): ''' Returns if the passed id is valid ''' try: - return bool(clean_path(opts['pki_dir'], id_)) and clean_id(id_) - except (AttributeError, KeyError, TypeError) as e: + if any(x in id_ for x in ('/', '\\', '\0')): + return False + return bool(clean_path(opts['pki_dir'], id_)) + except (AttributeError, KeyError, TypeError): return False diff --git a/tests/unit/utils/test_verify.py b/tests/unit/utils/test_verify.py index 795298877d..fa091f6cb3 100644 --- a/tests/unit/utils/test_verify.py +++ b/tests/unit/utils/test_verify.py @@ -58,6 +58,16 @@ class TestVerify(TestCase): opts = {'pki_dir': '/tmp/whatever'} self.assertFalse(valid_id(opts, None)) + def test_valid_id_pathsep(self): + ''' + Path separators in id should make it invalid + ''' + opts = {'pki_dir': '/tmp/whatever'} + # We have to test both path separators because os.path.normpath will + # convert forward slashes to backslashes on Windows. + for pathsep in ('/', '\\'): + self.assertFalse(valid_id(opts, pathsep.join(('..', 'foobar')))) + def test_zmq_verify(self): self.assertTrue(zmq_version()) From 206ae23f15cb7ec95a07dee4cbe9802da84f9c42 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 23 Aug 2017 10:20:50 -0500 Subject: [PATCH 026/184] Don't allow path separators in minion ID --- salt/utils/verify.py | 15 ++++----------- tests/unit/utils/verify_test.py | 10 ++++++++++ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/salt/utils/verify.py b/salt/utils/verify.py index db513ba675..45581f02ce 100644 --- a/salt/utils/verify.py +++ b/salt/utils/verify.py @@ -481,22 +481,15 @@ def clean_path(root, path, subdir=False): return '' -def clean_id(id_): - ''' - Returns if the passed id is clean. - ''' - if re.search(r'\.\.{sep}'.format(sep=os.sep), id_): - return False - return True - - def valid_id(opts, id_): ''' Returns if the passed id is valid ''' try: - return bool(clean_path(opts['pki_dir'], id_)) and clean_id(id_) - except (AttributeError, KeyError) as e: + if any(x in id_ for x in ('/', '\\', '\0')): + return False + return bool(clean_path(opts['pki_dir'], id_)) + except (AttributeError, KeyError, TypeError): return False diff --git a/tests/unit/utils/verify_test.py b/tests/unit/utils/verify_test.py index 7e60f886d0..c3fa373290 100644 --- a/tests/unit/utils/verify_test.py +++ b/tests/unit/utils/verify_test.py @@ -60,6 +60,16 @@ class TestVerify(TestCase): opts = {'pki_dir': '/tmp/whatever'} self.assertFalse(valid_id(opts, None)) + def test_valid_id_pathsep(self): + ''' + Path separators in id should make it invalid + ''' + opts = {'pki_dir': '/tmp/whatever'} + # We have to test both path separators because os.path.normpath will + # convert forward slashes to backslashes on Windows. + for pathsep in ('/', '\\'): + self.assertFalse(valid_id(opts, pathsep.join(('..', 'foobar')))) + def test_zmq_verify(self): self.assertTrue(zmq_version()) From 89e084bda356739de645c15e7d1968afebdcc56e Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 25 Aug 2017 14:15:58 -0500 Subject: [PATCH 027/184] Do not allow IDs with null bytes in decoded payloads --- salt/crypt.py | 3 +++ salt/transport/tcp.py | 11 +++++++++++ salt/transport/zeromq.py | 11 +++++++++++ 3 files changed, 25 insertions(+) diff --git a/salt/crypt.py b/salt/crypt.py index d330594a2a..46c3329741 100644 --- a/salt/crypt.py +++ b/salt/crypt.py @@ -606,6 +606,9 @@ class AsyncAuth(object): raise tornado.gen.Return('retry') else: raise SaltClientError('Attempt to authenticate with the salt master failed with timeout error') + if not isinstance(payload, dict): + log.error('Sign-in attempt failed: %s', payload) + raise tornado.gen.Return(False) if 'load' in payload: if 'ret' in payload['load']: if not payload['load']['ret']: diff --git a/salt/transport/tcp.py b/salt/transport/tcp.py index a9001f03a5..f274240a1e 100644 --- a/salt/transport/tcp.py +++ b/salt/transport/tcp.py @@ -623,6 +623,17 @@ class TCPReqServerChannel(salt.transport.mixins.auth.AESReqServerMixin, salt.tra 'payload and load must be a dict', header=header)) raise tornado.gen.Return() + try: + id_ = payload['load'].get('id', '') + if '\0' in id_: + log.error('Payload contains an id with a null byte: %s', payload) + stream.send(self.serial.dumps('bad load: id contains a null byte')) + raise tornado.gen.Return() + except TypeError: + log.error('Payload contains non-string id: %s', payload) + stream.send(self.serial.dumps('bad load: id {0} is not a string'.format(id_))) + raise tornado.gen.Return() + # intercept the "_auth" commands, since the main daemon shouldn't know # anything about our key auth if payload['enc'] == 'clear' and payload.get('load', {}).get('cmd') == '_auth': diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py index 94aeb3e21a..eed012aec7 100644 --- a/salt/transport/zeromq.py +++ b/salt/transport/zeromq.py @@ -596,6 +596,17 @@ class ZeroMQReqServerChannel(salt.transport.mixins.auth.AESReqServerMixin, salt. stream.send(self.serial.dumps('payload and load must be a dict')) raise tornado.gen.Return() + try: + id_ = payload['load'].get('id', '') + if '\0' in id_: + log.error('Payload contains an id with a null byte: %s', payload) + stream.send(self.serial.dumps('bad load: id contains a null byte')) + raise tornado.gen.Return() + except TypeError: + log.error('Payload contains non-string id: %s', payload) + stream.send(self.serial.dumps('bad load: id {0} is not a string'.format(id_))) + raise tornado.gen.Return() + # intercept the "_auth" commands, since the main daemon shouldn't know # anything about our key auth if payload['enc'] == 'clear' and payload.get('load', {}).get('cmd') == '_auth': From 18fb0be96a146589ccbd642caa9244480c51140b Mon Sep 17 00:00:00 2001 From: Matthew Summers Date: Mon, 9 Oct 2017 20:38:52 -0500 Subject: [PATCH 028/184] addresses issue #43307, disk.format_ to disk.format This change fixes breakage. It appears the disk.format_ func is aliased to disk.format in modules/disk.py --- salt/states/blockdev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/blockdev.py b/salt/states/blockdev.py index 4b0dc5ca81..e6ecfeab3f 100644 --- a/salt/states/blockdev.py +++ b/salt/states/blockdev.py @@ -159,7 +159,7 @@ def formatted(name, fs_type='ext4', force=False, **kwargs): ret['result'] = None return ret - __salt__['disk.format_'](name, fs_type, force=force, **kwargs) + __salt__['disk.format'](name, fs_type, force=force, **kwargs) # Repeat fstype check up to 10 times with 3s sleeping between each # to avoid detection failing although mkfs has succeeded From 02986140b87abde2c5f1c1b7abbb33bbadc06821 Mon Sep 17 00:00:00 2001 From: Vernon Cole Date: Wed, 11 Oct 2017 04:41:20 -0600 Subject: [PATCH 029/184] utility support for vagrant-cloud --- salt/modules/vagrant.py | 24 +++++++++++++++++------- salt/utils/cloud.py | 5 +++-- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/salt/modules/vagrant.py b/salt/modules/vagrant.py index f94fd01c82..357362d204 100644 --- a/salt/modules/vagrant.py +++ b/salt/modules/vagrant.py @@ -54,7 +54,6 @@ def __virtual__(): ''' run Vagrant commands if possible ''' - # noinspection PyUnresolvedReferences if salt.utils.path.which('vagrant') is None: return False, 'The vagrant module could not be loaded: vagrant command not found' return __virtualname__ @@ -297,6 +296,11 @@ def vm_state(name='', cwd=None): 'provider': _, # the Vagrant VM provider 'name': _} # salt_id name + Known bug: if there are multiple machines in your Vagrantfile, and you request + the status of the ``primary`` machine, which you defined by leaving the ``machine`` + parameter blank, then you may receive the status of all of them. + Please specify the actual machine name for each VM if there are more than one. + ''' if name: @@ -320,7 +324,7 @@ def vm_state(name='', cwd=None): datum = {'machine': tokens[0], 'state': ' '.join(tokens[1:-1]), 'provider': tokens[-1].lstrip('(').rstrip(')'), - 'name': name or get_machine_id(tokens[0], cwd) + 'name': get_machine_id(tokens[0], cwd) } info.append(datum) except IndexError: @@ -364,7 +368,7 @@ def init(name, # Salt_id for created VM # passed-in keyword arguments overwrite vm dictionary values vm_['cwd'] = cwd or vm_.get('cwd') if not vm_['cwd']: - raise SaltInvocationError('Path to Vagrantfile must be defined by \'cwd\' argument') + raise SaltInvocationError('Path to Vagrantfile must be defined by "cwd" argument') vm_['machine'] = machine or vm_.get('machine', machine) vm_['runas'] = runas or vm_.get('runas', runas) vm_['vagrant_provider'] = vagrant_provider or vm_.get('vagrant_provider', '') @@ -422,7 +426,7 @@ def shutdown(name): ''' Send a soft shutdown (vagrant halt) signal to the named vm. - This does the same thing as vagrant.stop. Other VM control + This does the same thing as vagrant.stop. Other-VM control modules use "stop" and "shutdown" to differentiate between hard and soft shutdowns. @@ -475,20 +479,26 @@ def pause(name): return ret == 0 -def reboot(name): +def reboot(name, **kwargs): ''' Reboot a VM. (vagrant reload) + keyword argument: + + - provision: (False) also re-run the Vagrant provisioning scripts. + CLI Example: .. code-block:: bash - salt vagrant.reboot + salt vagrant.reboot provision=True ''' vm_ = get_vm_info(name) machine = vm_['machine'] + prov = kwargs.get('provision', False) + provision = '--provision' if prov else '' - cmd = 'vagrant reload {}'.format(machine) + cmd = 'vagrant reload {} {}'.format(machine, provision) ret = __salt__['cmd.retcode'](cmd, runas=vm_.get('runas'), cwd=vm_.get('cwd')) diff --git a/salt/utils/cloud.py b/salt/utils/cloud.py index b013c48f30..93be5715ae 100644 --- a/salt/utils/cloud.py +++ b/salt/utils/cloud.py @@ -397,13 +397,14 @@ def bootstrap(vm_, opts=None): # NOTE: deploy_kwargs is also used to pass inline_script variable content # to run_inline_script function + host = salt.config.get_cloud_config_value('ssh_host', vm_, opts) deploy_kwargs = { 'opts': opts, - 'host': vm_['ssh_host'], + 'host': host, 'port': salt.config.get_cloud_config_value( 'ssh_port', vm_, opts, default=22 ), - 'salt_host': vm_.get('salt_host', vm_['ssh_host']), + 'salt_host': vm_.get('salt_host', host), 'username': ssh_username, 'script': deploy_script_code, 'inline_script': inline_script_config, From 5b1069809c475a990e063ab7fc40b987b5bb7ee3 Mon Sep 17 00:00:00 2001 From: Vernon Cole Date: Wed, 11 Oct 2017 04:47:16 -0600 Subject: [PATCH 030/184] vagrant states --- salt/states/vagrant.py | 368 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 368 insertions(+) create mode 100644 salt/states/vagrant.py diff --git a/salt/states/vagrant.py b/salt/states/vagrant.py new file mode 100644 index 0000000000..f3f6b097b0 --- /dev/null +++ b/salt/states/vagrant.py @@ -0,0 +1,368 @@ +# -*- coding: utf-8 -*- +r''' +.. index:: Vagrant state function + +Manage Vagrant VMs +================== + +Manange execution of Vagrant virtual machines on Salt minions. + +Vagrant_ is a tool for building and managing virtual machine environments. +It can use various providers, such as VirtualBox_, Docker_, or VMware_, to run its VMs. +Vagrant provides some of the functionality of a light-weight hypervisor. +The combination of Salt modules, Vagrant running on the host, and a +virtual machine provider, gives hypervisor-like functionality for +developers who use Vagrant to quickly define their virtual environments. + +.. _Vagrant: http://www.vagrantup.com/ +.. _VirtualBox: https://www.virtualbox.org/ +.. _Docker: https://www.docker.io/ +.. _VMWare: https://www.vmware.com/ + + .. versionadded:: Oxygen + +The configuration of each virtual machine is defined in a file named +``Vagrantfile`` which must exist on the VM host machine. +The essential parameters which must be defined to start a Vagrant VM +are the directory where the ``Vagrantfile`` is located \(argument ``cwd:``\), +and the username which will own the ``Vagrant box`` created for the VM \( +argument ``vagrant_runas:``\). + +A single ``Vagrantfile`` may define one or more virtual machines. +Use the ``machine`` argument to chose among them. The default (blank) +value will select the ``primary`` (or only) machine in the Vagrantfile. + +\[NOTE:\] Each virtual machine host must have the following: + +- a working salt-minion +- a Salt sdb database configured for ``vagrant_sdb_data``. +- Vagrant installed and the ``vagrant`` command working +- a suitable VM provider + +.. code-block:: yaml + + # EXAMPLE: + # file /etc/salt/minion.d/vagrant_sdb.conf on the host computer + # -- this sdb database is required by the Vagrant module -- + vagrant_sdb_data: # The sdb database must have this name. + driver: sqlite3 # Let's use SQLite to store the data ... + database: /var/cache/salt/vagrant.sqlite # ... in this file ... + table: sdb # ... using this table name. + create_table: True # if not present + +''' +from __future__ import absolute_import + +# Import Python libs +import fnmatch + +# Import Salt libs +import salt.utils.args +from salt.exceptions import CommandExecutionError, SaltInvocationError +import salt.ext.six as six + +__virtualname__ = 'vagrant' + + +def __virtual__(): + ''' + Only if vagrant module is available. + + :return: + ''' + + if 'vagrant.version' in __salt__: + return __virtualname__ + return False + + +def _vagrant_call(node, function, section, comment, status_when_done=None,**kwargs): + ''' + Helper to call the vagrant functions. Wildcards supported. + + :param node: The Salt-id or wildcard + :param function: the vagrant submodule to call + :param section: the name for the state call. + :param comment: what the state reply should say + :param status_when_done: the Vagrant status expected for this state + :return: the dictionary for the state reply + ''' + ret = {'name': node, 'changes': {}, 'result': True, 'comment': ''} + + targeted_nodes = [] + if isinstance(node, six.string_types): + try: # use shortcut if a single node name + if __salt__['vagrant.get_vm_info'](node): + targeted_nodes = [node] + except (SaltInvocationError): + pass + + if not targeted_nodes: # the shortcut failed, do this the hard way + all_domains = __salt__['vagrant.list_domains']() + targeted_nodes = fnmatch.filter(all_domains, node) + changed_nodes = [] + ignored_nodes = [] + for node in targeted_nodes: + if status_when_done: + try: + present_state = __salt__['vagrant.vm_state'](node)[0] + if present_state['state'] == status_when_done: + continue # no change is needed + except (IndexError, SaltInvocationError, CommandExecutionError): + pass + try: + response = __salt__['vagrant.{0}'.format(function)](node, **kwargs) + if isinstance(response, dict): + response = response['name'] + changed_nodes.append({'node': node, function: response}) + except (SaltInvocationError, CommandExecutionError) as err: + ignored_nodes.append({'node': node, 'issue': str(err)}) + if not changed_nodes: + ret['result'] = True + ret['comment'] = 'No changes seen' + if ignored_nodes: + ret['changes'] = {'ignored': ignored_nodes} + else: + ret['changes'] = {section: changed_nodes} + ret['comment'] = comment + + return ret + + +def running(name, **kwargs): + r''' + Defines and starts a new VM with specified arguments, or restart a + VM (or group of VMs). (Runs ``vagrant up``.) + + :param name: the Salt_id node name you wish your VM to have. + + If ``name`` contains a "?" or "*" then it will re-start a group of VMs + which have been paused or stopped. + + Each machine must be initially started individually using this function + or the vagrant.init execution module call. + + \[NOTE:\] Keyword arguments are silently ignored when re-starting an existing VM. + + Possible keyword arguments: + + - cwd: The directory (path) containing the Vagrantfile + - machine: ('') the name of the machine (in the Vagrantfile) if not default + - vagrant_runas: ('root') the username who owns the vagrantbox file + - vagrant_provider: the provider to run the VM (usually 'virtualbox') + - vm: ({}) a dictionary containing these or other keyword arguments + + .. code-block:: yaml + + node_name: + vagrant.running + + .. code-block:: yaml + + node_name: + vagrant.running: + - cwd: /projects/my_project + - vagrant_runas: my_username + - machine: machine1 + + ''' + if '*' in name or '?' in name: + + return _vagrant_call(name, 'start', 'restarted', + "Machine has been restarted", "running") + + else: + + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': '{0} is already running'.format(name) + } + + try: + info = __salt__['vagrant.vm_state'](name) + if info[0]['state'] != 'running': + __salt__['vagrant.start'](name) + ret['changes'][name] = 'Machine started' + ret['comment'] = 'Node {0} started'.format(name) + except (SaltInvocationError, CommandExecutionError): + # there was no viable existing machine to start + ret, kwargs = _find_init_change(name, ret, **kwargs) + kwargs['start'] = True + __salt__['vagrant.init'](name, **kwargs) + ret['changes'][name] = 'Node defined and started' + ret['comment'] = 'Node {0} defined and started'.format(name) + + return ret + + +def _find_init_change(name, ret, **kwargs): + ''' + look for changes from any previous init of machine. + + :return: modified ret and kwargs + ''' + kwargs = salt.utils.args.clean_kwargs(**kwargs) + if 'vm' in kwargs: + kwargs.update(kwargs.pop('vm')) + # the state processing eats 'runas' so we rename + kwargs['runas'] = kwargs.pop('vagrant_runas', '') + try: + vm_ = __salt__['vagrant.get_vm_info'](name) + except SaltInvocationError: + vm_ = {} + for key, value in kwargs.items(): + ret['changes'][key] = {'old': None, 'new': value} + if vm_: # test for changed values + for key in vm_: + value = vm_[key] or '' # supply a blank if value is None + if key != 'name': # will be missing in kwargs + new = kwargs.get(key, '') + if new != value: + if key == 'machine' and new == '': + continue # we don't know the default machine name + ret['changes'][key] = {'old': value, 'new': new} + return ret, kwargs + + +def initialized(name, **kwargs): + r''' + Defines a new VM with specified arguments, but does not start it. + + :param name: the Salt_id node name you wish your VM to have. + + Each machine must be initialized individually using this function + or the "vagrant.running" function, or the vagrant.init execution module call. + + This command will not change the state of a running or paused machine. + + Possible keyword arguments: + + - cwd: The directory (path) containing the Vagrantfile + - machine: ('') the name of the machine (in the Vagrantfile) if not default + - vagrant_runas: ('root') the username who owns the vagrantbox file + - vagrant_provider: the provider to run the VM (usually 'virtualbox') + - vm: ({}) a dictionary containing these or other keyword arguments + + .. code-block:: yaml + + node_name1: + vagrant.initialized + - cwd: /projects/my_project + - vagrant_runas: my_username + - machine: machine1 + + node_name2: + vagrant.initialized + - cwd: /projects/my_project + - vagrant_runas: my_username + - machine: machine2 + + start_nodes: + vagrant.start: + - name: node_name? + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': 'The VM is already correctly defined' + } + + # define a machine to start later + ret, kwargs = _find_init_change(name, ret, **kwargs) + + if ret['changes'] == {}: + return ret + + kwargs['start'] = False + __salt__['vagrant.init'](name, **kwargs) + ret['changes'][name] = 'Node initialized' + ret['comment'] = 'Node {0} defined but not started.'.format(name) + + return ret + +def stopped(name): + ''' + Stops a VM (or VMs) by shutting it (them) down nicely. (Runs ``vagrant halt``) + + :param name: May be a Salt_id node, or a POSIX-style wildcard string. + + .. code-block:: yaml + + node_name: + vagrant.stopped + ''' + + return _vagrant_call(name, 'shutdown', 'stopped', + 'Machine has been shut down', 'poweroff') + + +def powered_off(name): + ''' + Stops a VM (or VMs) by power off. (Runs ``vagrant halt``.) + + This method is provided for compatibility with other VM-control + state modules. For Vagrant, the action is identical with ``stopped``. + + :param name: May be a Salt_id node or a POSIX-style wildcard string. + + .. code-block:: yaml + + node_name: + vagrant.unpowered + ''' + + return _vagrant_call(name, 'stop', 'unpowered', + 'Machine has been powered off', 'poweroff') + + +def destroyed(name): + ''' + Stops a VM (or VMs) and removes all refences to it (them). (Runs ``vagrant destroy``.) + + Subsequent re-use of the same machine will requere another operation of ``vagrant.running`` + or a call to the ``vagrant.init`` execution module. + + :param name: May be a Salt_id node or a POSIX-style wildcard string. + + .. code-block:: yaml + + node_name: + vagrant.destroyed + ''' + + return _vagrant_call(name, 'destroy', 'destroyed', + 'Machine has been removed') + + +def paused(name): + ''' + Stores the state of a VM (or VMs) for fast restart. (Runs ``vagrant suspend``.) + + :param name: May be a Salt_id node or a POSIX-style wildcard string. + + .. code-block:: yaml + + node_name: + vagrant.paused + ''' + + return _vagrant_call(name, 'pause', 'paused', + 'Machine has been suspended', 'saved') + + +def rebooted(name): + ''' + Reboots a running, paused, or stopped VM (or VMs). (Runs ``vagrant reload``.) + + The will re-run the provisioning + + :param name: May be a Salt_id node or a POSIX-style wildcard string. + + .. code-block:: yaml + + node_name: + vagrant.reloaded + ''' + + return _vagrant_call(name, 'reboot', 'rebooted', 'Machine has been reloaded') From 41298e9c5741d39e71ec4b0e99ee426a8608119e Mon Sep 17 00:00:00 2001 From: Vernon Cole Date: Wed, 11 Oct 2017 04:48:09 -0600 Subject: [PATCH 031/184] vagrant cloud driver --- salt/cloud/__init__.py | 2 +- salt/cloud/clouds/vagrant.py | 339 +++++++++++++++++++++++++++++++++++ 2 files changed, 340 insertions(+), 1 deletion(-) create mode 100644 salt/cloud/clouds/vagrant.py diff --git a/salt/cloud/__init__.py b/salt/cloud/__init__.py index 0372f91f63..e5d05cab1c 100644 --- a/salt/cloud/__init__.py +++ b/salt/cloud/__init__.py @@ -1416,7 +1416,7 @@ class Cloud(object): if name in vms: prov = vms[name]['provider'] driv = vms[name]['driver'] - msg = six.u('{0} already exists under {1}:{2}').format( + msg = u'{0} already exists under {1}:{2}'.format( name, prov, driv ) log.error(msg) diff --git a/salt/cloud/clouds/vagrant.py b/salt/cloud/clouds/vagrant.py new file mode 100644 index 0000000000..462ec6ed01 --- /dev/null +++ b/salt/cloud/clouds/vagrant.py @@ -0,0 +1,339 @@ +# -*- coding: utf-8 -*- +''' +Vagrant Cloud Driver +==================== + +The Vagrant cloud is designed to "vagrant up" a virtual machine as a +Salt minion. + +Use of this module requires some configuration in cloud profile and provider +files as described in the +:ref:`Gettting Started with Vagrant ` documentation. + +.. versionadded:: Oxygen + + +''' + +# Import python libs +from __future__ import absolute_import +import logging +import os +import tempfile + +# Import salt libs +import salt.utils +import salt.config as config +import salt.client +import salt.ext.six as six +if six.PY3: + import ipaddress +else: + import salt.ext.ipaddress as ipaddress +from salt.exceptions import SaltCloudException, SaltCloudSystemExit + +# Get logging started +log = logging.getLogger(__name__) + + +def __virtual__(): + ''' + Needs no special configuration + ''' + return True + + +def avail_locations(call=None): + r''' + This function returns a list of locations available. + + CLI Example: + + .. code-block:: bash + + salt-cloud --list-locations my-cloud-provider + + # \[ vagrant will always returns an empty dictionary \] + + ''' + + return {} + + +def avail_images(call=None): + '''This function returns a list of images available for this cloud provider. + vagrant will return a list of profiles. + salt-cloud --list-images my-cloud-provider + ''' + vm_ = get_configured_provider() + return {'Profiles': [profile for profile in vm_['profiles']]} + + +def avail_sizes(call=None): + r''' + This function returns a list of sizes available for this cloud provider. + + CLI Example: + + .. code-block:: bash + + salt-cloud --list-sizes my-cloud-provider + + # \[ vagrant always returns an empty dictionary \] + + ''' + return {} + + +def list_nodes(call=None): + ''' + List the nodes which have salt-cloud:driver:vagrant grains. + + CLI Example: + + .. code-block:: bash + + salt-cloud -Q + ''' + nodes = _list_nodes(call) + return _build_required_items(nodes) + + +def _build_required_items(nodes): + ret = {} + for name, grains in nodes.items(): + if grains: + private_ips = [] + public_ips = [] + ips = grains['ipv4'] + grains['ipv6'] + for adrs in ips: + ip_ = ipaddress.ip_address(adrs) + if not ip_.is_loopback: + if ip_.is_private: + private_ips.append(adrs) + else: + public_ips.append(adrs) + + ret[name] = { + 'id': grains['id'], + 'image': grains['salt-cloud']['profile'], + 'private_ips': private_ips, + 'public_ips': public_ips, + 'size': '', + 'state': 'running' + } + + return ret + + +def list_nodes_full(call=None): + ''' + List the nodes, ask all 'vagrant' minions, return dict of grains (enhanced). + + CLI Example: + + .. code-block:: bash + + salt-call -F + ''' + ret = _list_nodes(call) + + for key, grains in ret.items(): # clean up some hyperverbose grains -- everything is too much + try: + del grains['cpu_flags'], grains['disks'], grains['pythonpath'], grains['dns'], grains['gpus'] + except KeyError: + pass # ignore absence of things we are eliminating + except TypeError: + del ret[key] # eliminate all reference to unexpected (None) values. + + reqs = _build_required_items(ret) + for name in ret: + ret[name].update(reqs[name]) + return ret + + +def _list_nodes(call=None): + ''' + List the nodes, ask all 'vagrant' minions, return dict of grains. + ''' + local = salt.client.LocalClient() + ret = local.cmd('salt-cloud:driver:vagrant', 'grains.items', '', tgt_type='grain') + return ret + + +def list_nodes_select(call=None): + ''' + Return a list of the minions that have salt-cloud grains, with + select fields. + ''' + return salt.utils.cloud.list_nodes_select( + list_nodes_full('function'), __opts__['query.selection'], call, + ) + + +def show_instance(name, call=None): + ''' + List the a single node, return dict of grains. + ''' + local = salt.client.LocalClient() + ret = local.cmd(name, 'grains.items', '') + reqs = _build_required_items(ret) + ret[name].update(reqs[name]) + return ret + + +def _get_my_info(name): + local = salt.client.LocalClient() + return local.cmd(name, 'grains.get', ['salt-cloud']) + + +def create(vm_): + ''' + Provision a single machine + + CLI Example: + + .. code-block:: bash + salt-cloud -p my_profile new_node_1 + + ''' + name = vm_['name'] + machine = config.get_cloud_config_value( + 'machine', vm_, __opts__, default='') + vm_['machine'] = machine + host = config.get_cloud_config_value( + 'host', vm_, __opts__, default=NotImplemented) + vm_['cwd'] = config.get_cloud_config_value( + 'cwd', vm_, __opts__, default='/') + vm_['runas'] = config.get_cloud_config_value( + 'vagrant_runas', vm_, __opts__, default=os.getenv('SUDO_USER')) + vm_['timeout'] = config.get_cloud_config_value( + 'vagrant_up_timeout', vm_, __opts__, default=300) + vm_['vagrant_provider'] = config.get_cloud_config_value( + 'vagrant_provider', vm_, __opts__, default='') + vm_['grains'] = {'salt-cloud:vagrant': {'host': host, 'machine': machine}} + + log.info('sending \'vagrant.init %s machine=%s\' command to %s', name, machine, host) + + local = salt.client.LocalClient() + ret = local.cmd(host, 'vagrant.init', [name], kwarg={'vm': vm_, 'start': True}) + log.info('response ==> %s', ret[host]) + + network_mask = config.get_cloud_config_value( + 'network_mask', vm_, __opts__, default='') + if 'ssh_host' not in vm_: + local = salt.client.LocalClient() + ret = local.cmd(host, + 'vagrant.get_ssh_config', + [name], + kwarg={'network_mask': network_mask, + 'get_private_key': True})[host] + with tempfile.NamedTemporaryFile() as pks: + if 'private_key' not in vm_ and ret.get('private_key', False): + pks.write(ret['private_key']) + pks.flush() + log.debug('wrote private key to %s', pks.name) + vm_['key_filename'] = pks.name + if 'ssh_host' not in vm_: + vm_.setdefault('ssh_username', ret['ssh_username']) + if ret.get('ip_address'): + vm_['ssh_host'] = ret['ip_address'] + else: # if probe failed or not used, use Vagrant's reported ssh info + vm_['ssh_host'] = ret['ssh_host'] + vm_.setdefault('ssh_port', ret['ssh_port']) + + log.info('Provisioning machine %s as node %s using ssh %s', + machine, name, vm_['ssh_host']) + ret = __utils__['cloud.bootstrap'](vm_, __opts__) + return ret + + +def get_configured_provider(): + ''' + Return the first configured instance. + ''' + ret = config.is_provider_configured( + __opts__, + __active_provider_name__ or 'vagrant', + '' + ) + return ret + + +# noinspection PyTypeChecker +def destroy(name, call=None): + ''' + Destroy a node. + + CLI Example: + + .. code-block:: bash + + salt-cloud --destroy mymachine + ''' + if call == 'function': + raise SaltCloudSystemExit( + 'The destroy action must be called with -d, --destroy, ' + '-a, or --action.' + ) + + opts = __opts__ + + __utils__['cloud.fire_event']( + 'event', + 'destroying instance', + 'salt/cloud/{0}/destroying'.format(name), + args={'name': name}, + sock_dir=opts['sock_dir'], + transport=opts['transport'] + ) + my_info = _get_my_info(name) + profile_name = my_info[name]['profile'] + profile = opts['profiles'][profile_name] + host = profile['host'] + local = salt.client.LocalClient() + ret = local.cmd(host, 'vagrant.destroy', [name]) + + if ret[host]: + __utils__['cloud.fire_event']( + 'event', + 'destroyed instance', + 'salt/cloud/{0}/destroyed'.format(name), + args={'name': name}, + sock_dir=opts['sock_dir'], + transport=opts['transport'] + ) + + if opts.get('update_cachedir', False) is True: + __utils__['cloud.delete_minion_cachedir']( + name, __active_provider_name__.split(':')[0], opts) + + return {'Destroyed': '{0} was destroyed.'.format(name)} + else: + return {'Error': 'Error destroying {}'.format(name)} + + +# noinspection PyTypeChecker +def reboot(name, call=None): + ''' + Reboot a vagrant minion. + + name + The name of the VM to reboot. + + CLI Example: + + .. code-block:: bash + + salt-cloud -a reboot vm_name + ''' + if call != 'action': + raise SaltCloudException( + 'The reboot action must be called with -a or --action.' + ) + my_info = _get_my_info(name) + profile_name = my_info[name]['profile'] + profile = __opts__['profiles'][profile_name] + host = profile['host'] + local = salt.client.LocalClient() + return local.cmd(host, 'vagrant.reboot', [name]) From 4cdb04b409c0e24ab48d4e2bc41f3df296dcc233 Mon Sep 17 00:00:00 2001 From: Vernon Cole Date: Wed, 11 Oct 2017 04:49:47 -0600 Subject: [PATCH 032/184] documentation for vagrant cloud & state --- doc/ref/clouds/all/index.rst | 1 + .../clouds/all/salt.cloud.clouds.vagrant.rst | 6 + doc/ref/states/all/index.rst | 1 + doc/ref/states/all/salt.states.vagrant.rst | 6 + doc/topics/cloud/config.rst | 11 + doc/topics/cloud/features.rst | 282 +++++++++--------- doc/topics/cloud/index.rst | 1 + doc/topics/cloud/saltify.rst | 2 + doc/topics/cloud/vagrant.rst | 268 +++++++++++++++++ 9 files changed, 440 insertions(+), 138 deletions(-) create mode 100644 doc/ref/clouds/all/salt.cloud.clouds.vagrant.rst create mode 100644 doc/ref/states/all/salt.states.vagrant.rst create mode 100644 doc/topics/cloud/vagrant.rst diff --git a/doc/ref/clouds/all/index.rst b/doc/ref/clouds/all/index.rst index 15fb4b1ae3..3ec40b0ed7 100644 --- a/doc/ref/clouds/all/index.rst +++ b/doc/ref/clouds/all/index.rst @@ -34,6 +34,7 @@ Full list of Salt Cloud modules scaleway softlayer softlayer_hw + vagrant virtualbox vmware vultrpy diff --git a/doc/ref/clouds/all/salt.cloud.clouds.vagrant.rst b/doc/ref/clouds/all/salt.cloud.clouds.vagrant.rst new file mode 100644 index 0000000000..ba3dcbe2d7 --- /dev/null +++ b/doc/ref/clouds/all/salt.cloud.clouds.vagrant.rst @@ -0,0 +1,6 @@ +========================= +salt.cloud.clouds.vagrant +========================= + +.. automodule:: salt.cloud.clouds.vagrant + :members: diff --git a/doc/ref/states/all/index.rst b/doc/ref/states/all/index.rst index 0b681ace7e..2fb3491b76 100644 --- a/doc/ref/states/all/index.rst +++ b/doc/ref/states/all/index.rst @@ -267,6 +267,7 @@ state modules tuned uptime user + vagrant vault vbox_guest victorops diff --git a/doc/ref/states/all/salt.states.vagrant.rst b/doc/ref/states/all/salt.states.vagrant.rst new file mode 100644 index 0000000000..5d5b6e9f9c --- /dev/null +++ b/doc/ref/states/all/salt.states.vagrant.rst @@ -0,0 +1,6 @@ +=================== +salt.states.vagrant +=================== + +.. automodule:: salt.states.vagrant + :members: \ No newline at end of file diff --git a/doc/topics/cloud/config.rst b/doc/topics/cloud/config.rst index 173ea4e692..e934a047d0 100644 --- a/doc/topics/cloud/config.rst +++ b/doc/topics/cloud/config.rst @@ -540,6 +540,17 @@ machines which are already installed, but not Salted. For more information about this driver and for configuration examples, please see the :ref:`Gettting Started with Saltify ` documentation. +.. _config_vagrant: + +Vagrant +------- + +The Vagrant driver is a new, experimental driver for controlling a VagrantBox +virtual machine, and installing Salt on it. The target host machine must be a +working salt minion, which is controlled via the salt master using salt-api. +For more information, see +:ref:`Getting Started With Vagrant `. + Extending Profiles and Cloud Providers Configuration ==================================================== diff --git a/doc/topics/cloud/features.rst b/doc/topics/cloud/features.rst index b067dc9a30..f270198dee 100644 --- a/doc/topics/cloud/features.rst +++ b/doc/topics/cloud/features.rst @@ -38,26 +38,30 @@ These are features that are available for almost every cloud host. .. container:: scrollable - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - | |AWS |CloudStack|Digital|EC2|GoGrid|JoyEnt|Linode|OpenStack|Parallels|Rackspace|Saltify|Softlayer|Softlayer|Aliyun| - | |(Legacy)| |Ocean | | | | | | |(Legacy) | | |Hardware | | - +=======================+========+==========+=======+===+======+======+======+=========+=========+=========+=======+=========+=========+======+ - |Query |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes | |Yes |Yes |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |Full Query |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes | |Yes |Yes |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |Selective Query |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes | |Yes |Yes |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |List Sizes |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes | |Yes |Yes |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |List Images |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes | |Yes |Yes |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |List Locations |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes | |Yes |Yes |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |create |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |destroy |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes | |Yes |Yes |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+-------+---------+---------+------+ + | |AWS |CloudStack|Digital|EC2|GoGrid|JoyEnt|Linode|OpenStack|Parallels|Rackspace|Saltify|Vagrant|Softlayer|Softlayer|Aliyun| + | |(Legacy)| |Ocean | | | | | | |(Legacy) | | | |Hardware | | + +=======================+========+==========+=======+===+======+======+======+=========+=========+=========+=======+=======+=========+=========+======+ + |Query |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes |[1] |[1] |Yes |Yes |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+-------+---------+---------+------+ + |Full Query |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes |[1] |[1] |Yes |Yes |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+-------+---------+---------+------+ + |Selective Query |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes |[1] |[1] |Yes |Yes |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+-------+---------+---------+------+ + |List Sizes |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes |[2] |[2] |Yes |Yes |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+-------+---------+---------+------+ + |List Images |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+-------+---------+---------+------+ + |List Locations |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes |[2] |[2] |Yes |Yes |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+-------+---------+---------+------+ + |create |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes |Yes |[1] |Yes |Yes |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+-------+---------+---------+------+ + |destroy |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes |[1] |[1] |Yes |Yes |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+-------+---------+---------+------+ + +[1] Yes, if salt-api is enabled. + +[2] Always returns `{}`. Actions ======= @@ -70,46 +74,46 @@ instance name to be passed in. For example: .. container:: scrollable - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |Actions |AWS |CloudStack|Digital|EC2|GoGrid|JoyEnt|Linode|OpenStack|Parallels|Rackspace|Saltify|Softlayer|Softlayer|Aliyun| - | |(Legacy)| |Ocean | | | | | | |(Legacy) | | |Hardware | | - +=======================+========+==========+=======+===+======+======+======+=========+=========+=========+=======+=========+=========+======+ - |attach_volume | | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |create_attach_volumes |Yes | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |del_tags |Yes | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |delvol_on_destroy | | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |detach_volume | | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |disable_term_protect |Yes | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |enable_term_protect |Yes | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |get_tags |Yes | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |keepvol_on_destroy | | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |list_keypairs | | |Yes | | | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |rename |Yes | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |set_tags |Yes | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |show_delvol_on_destroy | | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |show_instance | | |Yes |Yes| | |Yes | |Yes | | |Yes |Yes |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |show_term_protect | | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |start |Yes | | |Yes| |Yes |Yes | |Yes | | | | |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |stop |Yes | | |Yes| |Yes |Yes | |Yes | | | | |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |take_action | | | | | |Yes | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |Actions |AWS |CloudStack|Digital|EC2|GoGrid|JoyEnt|Linode|OpenStack|Parallels|Rackspace|Saltify&|Softlayer|Softlayer|Aliyun| + | |(Legacy)| |Ocean | | | | | | |(Legacy) | Vagrant| |Hardware | | + +=======================+========+==========+=======+===+======+======+======+=========+=========+=========+========+=========+=========+======+ + |attach_volume | | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |create_attach_volumes |Yes | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |del_tags |Yes | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |delvol_on_destroy | | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |detach_volume | | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |disable_term_protect |Yes | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |enable_term_protect |Yes | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |get_tags |Yes | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |keepvol_on_destroy | | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |list_keypairs | | |Yes | | | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |rename |Yes | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |set_tags |Yes | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |show_delvol_on_destroy | | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |show_instance | | |Yes |Yes| | |Yes | |Yes | | |Yes |Yes |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |show_term_protect | | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |start |Yes | | |Yes| |Yes |Yes | |Yes | | | | |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |stop |Yes | | |Yes| |Yes |Yes | |Yes | | | | |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |take_action | | | | | |Yes | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ Functions ========= @@ -122,81 +126,83 @@ require the name of the provider to be passed in. For example: .. container:: scrollable - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |Functions |AWS |CloudStack|Digital|EC2|GoGrid|JoyEnt|Linode|OpenStack|Parallels|Rackspace|Saltify|Softlayer|Softlayer|Aliyun| - | |(Legacy)| |Ocean | | | | | | |(Legacy) | | |Hardware | | - +=======================+========+==========+=======+===+======+======+======+=========+=========+=========+=======+=========+=========+======+ - |block_device_mappings |Yes | | | | | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |create_keypair | | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |create_volume | | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |delete_key | | | | | |Yes | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |delete_keypair | | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |delete_volume | | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |get_image | | |Yes | | |Yes | | |Yes | | | | |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |get_ip | |Yes | | | | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |get_key | |Yes | | | | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |get_keyid | | |Yes | | | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |get_keypair | |Yes | | | | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |get_networkid | |Yes | | | | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |get_node | | | | | |Yes | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |get_password | |Yes | | | | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |get_size | | |Yes | | |Yes | | | | | | | |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |get_spot_config | | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |get_subnetid | | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |iam_profile |Yes | | |Yes| | | | | | | | | |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |import_key | | | | | |Yes | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |key_list | | | | | |Yes | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |keyname |Yes | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |list_availability_zones| | | |Yes| | | | | | | | | |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |list_custom_images | | | | | | | | | | | |Yes | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |list_keys | | | | | |Yes | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |list_nodes |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |list_nodes_full |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |list_nodes_select |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |list_vlans | | | | | | | | | | | |Yes |Yes | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |rackconnect | | | | | | | |Yes | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |reboot | | | |Yes| |Yes | | | | | | | |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |reformat_node | | | | | |Yes | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |securitygroup |Yes | | |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |securitygroupid | | | |Yes| | | | | | | | | |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |show_image | | | |Yes| | | | |Yes | | | | |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |show_key | | | | | |Yes | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |show_keypair | | |Yes |Yes| | | | | | | | | | | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ - |show_volume | | | |Yes| | | | | | | | | |Yes | - +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+-------+---------+---------+------+ + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |Functions |AWS |CloudStack|Digital|EC2|GoGrid|JoyEnt|Linode|OpenStack|Parallels|Rackspace|Saltify&|Softlayer|Softlayer|Aliyun| + | |(Legacy)| |Ocean | | | | | | |(Legacy) | Vagrant| |Hardware | | + +=======================+========+==========+=======+===+======+======+======+=========+=========+=========+========+=========+=========+======+ + |block_device_mappings |Yes | | | | | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |create_keypair | | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |create_volume | | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |delete_key | | | | | |Yes | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |delete_keypair | | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |delete_volume | | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |get_image | | |Yes | | |Yes | | |Yes | | | | |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |get_ip | |Yes | | | | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |get_key | |Yes | | | | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |get_keyid | | |Yes | | | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |get_keypair | |Yes | | | | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |get_networkid | |Yes | | | | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |get_node | | | | | |Yes | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |get_password | |Yes | | | | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |get_size | | |Yes | | |Yes | | | | | | | |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |get_spot_config | | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |get_subnetid | | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |iam_profile |Yes | | |Yes| | | | | | | | | |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |import_key | | | | | |Yes | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |key_list | | | | | |Yes | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |keyname |Yes | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |list_availability_zones| | | |Yes| | | | | | | | | |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |list_custom_images | | | | | | | | | | | |Yes | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |list_keys | | | | | |Yes | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |list_nodes |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |list_nodes_full |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |list_nodes_select |Yes |Yes |Yes |Yes|Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |list_vlans | | | | | | | | | | | |Yes |Yes | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |rackconnect | | | | | | | |Yes | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |reboot | | | |Yes| |Yes | | | | |[1] | | |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |reformat_node | | | | | |Yes | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |securitygroup |Yes | | |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |securitygroupid | | | |Yes| | | | | | | | | |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |show_image | | | |Yes| | | | |Yes | | | | |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |show_key | | | | | |Yes | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |show_keypair | | |Yes |Yes| | | | | | | | | | | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + |show_volume | | | |Yes| | | | | | | | | |Yes | + +-----------------------+--------+----------+-------+---+------+------+------+---------+---------+---------+--------+---------+---------+------+ + +[1] Yes, if salt-api is enabled. diff --git a/doc/topics/cloud/index.rst b/doc/topics/cloud/index.rst index eff8e2aa8f..d95ca09269 100644 --- a/doc/topics/cloud/index.rst +++ b/doc/topics/cloud/index.rst @@ -129,6 +129,7 @@ Cloud Provider Specifics Getting Started With Scaleway Getting Started With Saltify Getting Started With SoftLayer + Getting Started With Vagrant Getting Started With Vexxhost Getting Started With Virtualbox Getting Started With VMware diff --git a/doc/topics/cloud/saltify.rst b/doc/topics/cloud/saltify.rst index dda9801522..8b59da5b0e 100644 --- a/doc/topics/cloud/saltify.rst +++ b/doc/topics/cloud/saltify.rst @@ -183,6 +183,8 @@ simple installation. ssh_username: vagrant # a user name which has passwordless sudo password: vagrant # on your target machine provider: my_saltify_provider + shutdown_on_destroy: true # halt the target on "salt-cloud -d" command + .. code-block:: yaml diff --git a/doc/topics/cloud/vagrant.rst b/doc/topics/cloud/vagrant.rst new file mode 100644 index 0000000000..8a67dfc818 --- /dev/null +++ b/doc/topics/cloud/vagrant.rst @@ -0,0 +1,268 @@ +.. _getting-started-with-vagrant: + +============================ +Getting Started With Vagrant +============================ + +The Vagrant driver is a new, experimental driver for spinning up a VagrantBox +virtual machine, and installing Salt on it. + +Dependencies +============ +The Vagrant driver itself has no external dependencies. + +The machine which will host the VagrantBox must be an already existing minion +of the cloud server's Salt master. +It must have Vagrant_ installed, and a Vagrant-compatible virtual machine engine, +such as VirtualBox_. +(Note: The Vagrant driver does not depend on the salt-cloud VirtualBox driver in any way.) + +.. _Vagrant: https://www.vagrantup.com/ +.. _VirtualBox: https://www.virtualbox.org/ + +\[Caution: The version of Vagrant packaged for ``apt install`` in Ubuntu 16.04 will not connect a bridged +network adapter correctly. Use a version downloaded directly from the web site.\] + +Include the Vagrant guest editions plugin: +``vagrant plugin install vagrant-vbguest``. + +Configuration +============= + +Configuration of the client virtual machine (using VirtualBox, VMware, etc) +will be done by Vagrant as specified in the Vagrantfile on the host machine. + +Salt-cloud will push the commands to install and provision a salt minion on +the virtual machine, so you need not (perhaps **should** not) provision salt +in your Vagrantfile, in most cases. + +If, however, your cloud master cannot open an ssh connection to the child VM, +you may **need** to let Vagrant provision the VM with Salt, and use some other +method (such as passing a pillar dictionary to the VM) to pass the master's +IP address to the VM. The VM can then attempt to reach the salt master in the +usual way for non-cloud minions. Specify the profile configuration argument +as ``deploy: False`` to prevent the cloud master from trying. + +.. code-block:: yaml + + # Note: This example is for /etc/salt/cloud.providers file or any file in + # the /etc/salt/cloud.providers.d/ directory. + + my-vagrant-config: + minion: + master: 111.222.333.444 + provider: vagrant + + +Because the Vagrant driver needs a place to store the mapping between the +node name you use for Salt commands and the Vagrantfile which controls the VM, +you must configure your salt minion as a Salt smb server. +(See `host provisioning example`_ below.) + +Profiles +======== + +Vagrant requires a profile to be configured for each machine that needs Salt +installed. The initial profile can be set up at ``/etc/salt/cloud.profiles`` +or in the ``/etc/salt/cloud.profiles.d/`` directory. + +Each profile requires a ``vagrantfile`` parameter. If the Vagrantfile has +definitions for `multiple machines`_ then you need a ``machine`` parameter, + +.. _`multiple machines`: https://www.vagrantup.com/docs/multi-machine/ + +Salt-cloud uses ssh to provision the minion. There must be a routable path +from the cloud master to the VM. Usually, you will want to use +a bridged network adapter for ssh. The address may not be known until +DHCP assigns it. If ``ssh_host`` is not defined, and ``target_network`` +is defined, the driver will attempt to read the address from the output +of an ``ifconfig`` command. Lacking either setting, +the driver will try to use the value Vagrant returns as its ``ssh_host``, +which will work only if the cloud master is running somewhere on the same host. + +The ``target_network`` setting should be used +to identify the IP network your bridged adapter is expected to appear on. +Use CIDR notation, like ``target_network: '2001:DB8::/32'`` +or ``target_network: '192.0.2.0/24'``. + +Profile configuration example: + +.. code-block:: yaml + + # /etc/salt/cloud.profiles.d/vagrant.conf + + vagrant-machine: + host: my-vhost # the Salt id of the virtual machine's host computer. + provider: my-vagrant-config + cwd: /srv/machines # the path to your Virtualbox file. + vagrant_runas: my-username # the username who defined the Vagrantbox on the host + # vagrant_up_timeout: 300 # (seconds) timeout for cmd.run of the "vagrant up" command + # vagrant_provider: '' # option for "vagrant up" like: "--provider vmware_fusion" + # ssh_host: None # "None" means try to find the routable ip address from "ifconfig" + # target_network: None # Expected CIDR address of your bridged network + # force_minion_config: false # Set "true" to re-purpose an existing VM + +The machine can now be created and configured with the following command: + +.. code-block:: bash + + salt-cloud -p vagrant-machine my-id + +This will create the machine specified by the cloud profile +``vagrant-machine``, and will give the machine the minion id of +``my-id``. If the cloud master is also the salt-master, its Salt +key will automatically be accepted on the master. + +Once a salt-minion has been successfully installed on the instance, connectivity +to it can be verified with Salt: + +.. code-block:: bash + + salt my-id test.ping + +.. _host provisioning example: + +Provisioning a Vagrant cloud host (example) +=========================================== + +In order to query or control minions it created, each host +minion needs to track the Salt node names associated with +any guest virtual machines on it. +It does that using a Salt sdb database. + +The Salt sdb is not configured by default. The following example shows a +simple installation. + +This example assumes: + +- you are on a large network using the 10.x.x.x IP address space +- your Salt master's Salt id is "bevymaster" +- it will also be your salt-cloud controller +- it is at hardware address 10.124.30.7 +- it is running a recent Debian family Linux (raspbian) +- your workstation is a Salt minion of bevymaster +- your workstation's minion id is "my_laptop" +- VirtualBox has been installed on "my_laptop" (apt install is okay) +- Vagrant was installed from vagrantup.com. (not the 16.04 Ubuntu apt) +- "my_laptop" has done "vagrant plugin install vagrant-vbguest" +- the VM you want to start is on "my_laptop" at "/home/my_username/Vagrantfile" + +.. code-block:: yaml + + # file /etc/salt/minion.d/vagrant_sdb.conf on host computer "my_laptop" + # -- this sdb database is required by the Vagrant module -- + vagrant_sdb_data: # The sdb database must have this name. + driver: sqlite3 # Let's use SQLite to store the data ... + database: /var/cache/salt/vagrant.sqlite # ... in this file ... + table: sdb # ... using this table name. + create_table: True # if not present + +Remember to re-start your minion after changing its configuration files... + + ``sudo systemctl restart salt-minion`` + +.. code-block:: ruby + + # -*- mode: ruby -*- + # file /home/my_username/Vagrantfile on host computer "my_laptop" + BEVY = "bevy1" + DOMAIN = BEVY + ".test" # .test is an ICANN reserved non-public TLD + + # must supply a list of names to avoid Vagrant asking for interactive input + def get_good_ifc() # try to find a working Ubuntu network adapter name + addr_infos = Socket.getifaddrs + addr_infos.each do |info| + a = info.addr + if a and a.ip? and not a.ip_address.start_with?("127.") + return info.name + end + end + return "eth0" # fall back to an old reliable name + end + + Vagrant.configure(2) do |config| + config.ssh.forward_agent = true # so you can use git ssh://... + + # add a bridged network interface. (try to detect name, then guess MacOS names, too) + interface_guesses = [get_good_ifc(), 'en0: Ethernet', 'en1: Wi-Fi (AirPort)'] + config.vm.network "public_network", bridge: interface_guesses + if ARGV[0] == "up" + puts "Trying bridge network using interfaces: #{interface_guesses}" + end + config.vm.provision "shell", inline: "ip address", run: "always" # make user feel good + + # . . . . . . . . . . . . Define machine QUAIL1 . . . . . . . . . . . . . . + config.vm.define "quail1", primary: true do |quail_config| + quail_config.vm.box = "boxesio/xenial64-standard" # a public VMware & Virtualbox box + quail_config.vm.hostname = "quail1." + DOMAIN # supply a name in our bevy + quail_config.vm.provider "virtualbox" do |v| + v.memory = 1024 # limit memory for the virtual box + v.cpus = 1 + v.linked_clone = true # make a soft copy of the base Vagrant box + v.customize ["modifyvm", :id, "--natnet1", "192.168.128.0/24"] # do not use 10.x network for NAT + end + end + end + +.. code-block:: yaml + + # file /etc/salt/cloud.profiles.d/my_vagrant_profiles.conf on bevymaster + q1: + host: my_laptop # the Salt id of your virtual machine host + machine: quail1 # a machine name in the Vagrantfile (if not primary) + vagrant_runas: my_username # owner of Vagrant box files on "my_laptop" + cwd: '/home/my_username' # the path (on "my_laptop") of the Vagrantfile + provider: my_vagrant_provider # name of entry in provider.conf file + target_network: '10.0.0.0/8' # VM external address will be somewhere here + +.. code-block:: yaml + + # file /etc/salt/cloud.providers.d/vagrant_provider.conf on bevymaster + my_vagrant_provider: + driver: vagrant + minion: + master: 10.124.30.7 # the hard address of the master + + +Create and use your new Salt minion +----------------------------------- + +- Typing on the Salt master computer ``bevymaster``, tell it to create a new minion named ``v1`` using profile ``q1``... + +.. code-block:: bash + + sudo salt-cloud -p q1 v1 + sudo salt v1 network.ip_addrs + [ you get a list of ip addresses, including the bridged one ] + +- logged in to your laptop (or some other computer known to github)... + + \[NOTE:\] if you are using MacOS, you need to type ``ssh-add -K`` after each boot, + unless you use one of the methods in `this gist`_. + +.. _this gist: https://github.com/jirsbek/SSH-keys-in-macOS-Sierra-keychain + +.. code-block:: bash + + ssh -A vagrant@< the bridged network address > + # [ or, if you are at /home/my_username/ on my_laptop ] + vagrant ssh quail1 + +- then typing on your new node "v1" (a.k.a. quail1.bevy1.test)... + +.. code-block:: bash + + password: vagrant + # [ stuff types out ... ] + + ls -al /vagrant + # [ should be shared /home/my_username from my_laptop ] + + # you can access other network facilities using the ssh authorization + # as recorded in your ~.ssh/ directory on my_laptop ... + + sudo apt update + sudo apt install git + git clone ssh://git@github.com/yourID/your_project + # etc... + From e9c7c69402613bf7494377eb30b74ebc3ae4a591 Mon Sep 17 00:00:00 2001 From: Vernon Cole Date: Wed, 11 Oct 2017 05:02:26 -0600 Subject: [PATCH 033/184] documentation for vagrant cloud & state --- doc/topics/releases/oxygen.rst | 50 ++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/doc/topics/releases/oxygen.rst b/doc/topics/releases/oxygen.rst index 3c99a58ba1..50638195d2 100644 --- a/doc/topics/releases/oxygen.rst +++ b/doc/topics/releases/oxygen.rst @@ -132,6 +132,56 @@ file. For example: These commands will run in sequence **before** the bootstrap script is executed. +New salt-cloud Grains +===================== + +When salt cloud creates a new minon, it will now add grain information +to the minion configuration file, identifying the resources originally used +to create it. + +The generated grain information will appear similar to: + +.. code-block:: yaml + grains: + salt-cloud: + driver: ec2 + provider: my_ec2:ec2 + profile: ec2-web +The generation of salt-cloud grains can be surpressed by the +option ``enable_cloud_grains: 'False'`` in the cloud configuration file. + +Upgraded Saltify Driver +======================= + +The salt-cloud Saltify driver is used to provision machines which +are not controlled by a dedicated cloud supervisor (such as typical hardware +machines) by pushing a salt-bootstrap command to them and accepting them on +the salt master. Creation of a node has been its only function and no other +salt-cloud commands were implemented. + +With this upgrade, it can use the salt-api to provide advanced control, +such as rebooting a machine, querying it along with conventional cloud minions, +and, ultimately, disconnecting it from its master. + +After disconnection from ("destroying" on) one master, a machine can be +re-purposed by connecting to ("creating" on) a subsequent master. + +New Vagrant Driver +================== + +The salt-cloud Vagrant driver brings virtual machines running in a limited +environment, such as a programmer's workstation, under salt-cloud control. +This can be useful for experimentation, instruction, or testing salt configurations. + +Using salt-api on the master, and a salt-minion running on the host computer, +the Vagrant driver can create (``vagrant up``), restart (``vagrant reload``), +and destroy (``vagrant destroy``) VMs, as controlled by salt-cloud profiles +which designate a ``Vagrantfile`` on the host machine. + +The master can be a very limited machine, such as a Raspberry Pi, or a small +VagrantBox VM. + + New pillar/master_tops module called saltclass ---------------------------------------------- From 255aa94c6499646cfc0938822c7ba6dea006205c Mon Sep 17 00:00:00 2001 From: Nasenbaer Date: Mon, 17 Oct 2016 14:40:47 +0200 Subject: [PATCH 034/184] Activate jid_queue also for SingleMinions to workaround 0mq reconnection issues --- salt/minion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/minion.py b/salt/minion.py index 419d842be8..2cafd9ded6 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -935,7 +935,7 @@ class Minion(MinionBase): # Flag meaning minion has finished initialization including first connect to the master. # True means the Minion is fully functional and ready to handle events. self.ready = False - self.jid_queue = jid_queue + self.jid_queue = jid_queue or [] if io_loop is None: if HAS_ZMQ: From 42c0c6e6ef14d7702d743106dfaecf3a0448cf87 Mon Sep 17 00:00:00 2001 From: Vernon Cole Date: Thu, 12 Oct 2017 13:11:12 -0600 Subject: [PATCH 035/184] lint and documentation fixes --- doc/topics/cloud/vagrant.rst | 12 ++++++------ salt/cloud/clouds/vagrant.py | 3 +-- salt/modules/vagrant.py | 14 ++++++-------- salt/states/vagrant.py | 9 +++++---- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/doc/topics/cloud/vagrant.rst b/doc/topics/cloud/vagrant.rst index 8a67dfc818..466544e4b3 100644 --- a/doc/topics/cloud/vagrant.rst +++ b/doc/topics/cloud/vagrant.rst @@ -36,7 +36,7 @@ Salt-cloud will push the commands to install and provision a salt minion on the virtual machine, so you need not (perhaps **should** not) provision salt in your Vagrantfile, in most cases. -If, however, your cloud master cannot open an ssh connection to the child VM, +If, however, your cloud master cannot open an SSH connection to the child VM, you may **need** to let Vagrant provision the VM with Salt, and use some other method (such as passing a pillar dictionary to the VM) to pass the master's IP address to the VM. The VM can then attempt to reach the salt master in the @@ -71,9 +71,9 @@ definitions for `multiple machines`_ then you need a ``machine`` parameter, .. _`multiple machines`: https://www.vagrantup.com/docs/multi-machine/ -Salt-cloud uses ssh to provision the minion. There must be a routable path +Salt-cloud uses SSH to provision the minion. There must be a routable path from the cloud master to the VM. Usually, you will want to use -a bridged network adapter for ssh. The address may not be known until +a bridged network adapter for SSH. The address may not be known until DHCP assigns it. If ``ssh_host`` is not defined, and ``target_network`` is defined, the driver will attempt to read the address from the output of an ``ifconfig`` command. Lacking either setting, @@ -98,7 +98,7 @@ Profile configuration example: vagrant_runas: my-username # the username who defined the Vagrantbox on the host # vagrant_up_timeout: 300 # (seconds) timeout for cmd.run of the "vagrant up" command # vagrant_provider: '' # option for "vagrant up" like: "--provider vmware_fusion" - # ssh_host: None # "None" means try to find the routable ip address from "ifconfig" + # ssh_host: None # "None" means try to find the routable IP address from "ifconfig" # target_network: None # Expected CIDR address of your bridged network # force_minion_config: false # Set "true" to re-purpose an existing VM @@ -233,9 +233,9 @@ Create and use your new Salt minion sudo salt-cloud -p q1 v1 sudo salt v1 network.ip_addrs - [ you get a list of ip addresses, including the bridged one ] + [ you get a list of IP addresses, including the bridged one ] -- logged in to your laptop (or some other computer known to github)... +- logged in to your laptop (or some other computer known to GitHub)... \[NOTE:\] if you are using MacOS, you need to type ``ssh-add -K`` after each boot, unless you use one of the methods in `this gist`_. diff --git a/salt/cloud/clouds/vagrant.py b/salt/cloud/clouds/vagrant.py index 462ec6ed01..08fec40997 100644 --- a/salt/cloud/clouds/vagrant.py +++ b/salt/cloud/clouds/vagrant.py @@ -219,10 +219,9 @@ def create(vm_): ret = local.cmd(host, 'vagrant.init', [name], kwarg={'vm': vm_, 'start': True}) log.info('response ==> %s', ret[host]) - network_mask = config.get_cloud_config_value( + network_mask = config.get_cloud_config_value( 'network_mask', vm_, __opts__, default='') if 'ssh_host' not in vm_: - local = salt.client.LocalClient() ret = local.cmd(host, 'vagrant.get_ssh_config', [name], diff --git a/salt/modules/vagrant.py b/salt/modules/vagrant.py index 357362d204..7947a9b638 100644 --- a/salt/modules/vagrant.py +++ b/salt/modules/vagrant.py @@ -479,26 +479,24 @@ def pause(name): return ret == 0 -def reboot(name, **kwargs): +def reboot(name, provision=False): ''' Reboot a VM. (vagrant reload) - keyword argument: - - - provision: (False) also re-run the Vagrant provisioning scripts. - CLI Example: .. code-block:: bash salt vagrant.reboot provision=True + + :param name: The salt_id name you will use to control this VM + :param provision: (False) also re-run the Vagrant provisioning scripts. ''' vm_ = get_vm_info(name) machine = vm_['machine'] - prov = kwargs.get('provision', False) - provision = '--provision' if prov else '' + prov = '--provision' if provision else '' - cmd = 'vagrant reload {} {}'.format(machine, provision) + cmd = 'vagrant reload {} {}'.format(machine, prov) ret = __salt__['cmd.retcode'](cmd, runas=vm_.get('runas'), cwd=vm_.get('cwd')) diff --git a/salt/states/vagrant.py b/salt/states/vagrant.py index f3f6b097b0..edeec1f0db 100644 --- a/salt/states/vagrant.py +++ b/salt/states/vagrant.py @@ -76,7 +76,7 @@ def __virtual__(): return False -def _vagrant_call(node, function, section, comment, status_when_done=None,**kwargs): +def _vagrant_call(node, function, section, comment, status_when_done=None, **kwargs): ''' Helper to call the vagrant functions. Wildcards supported. @@ -94,7 +94,7 @@ def _vagrant_call(node, function, section, comment, status_when_done=None,**kwar try: # use shortcut if a single node name if __salt__['vagrant.get_vm_info'](node): targeted_nodes = [node] - except (SaltInvocationError): + except SaltInvocationError: pass if not targeted_nodes: # the shortcut failed, do this the hard way @@ -281,12 +281,13 @@ def initialized(name, **kwargs): return ret + def stopped(name): ''' Stops a VM (or VMs) by shutting it (them) down nicely. (Runs ``vagrant halt``) - + :param name: May be a Salt_id node, or a POSIX-style wildcard string. - + .. code-block:: yaml node_name: From b6b12fe49556f8685d44499f5c3eb004f1040207 Mon Sep 17 00:00:00 2001 From: Sergey Kizunov Date: Thu, 12 Oct 2017 16:08:21 -0500 Subject: [PATCH 036/184] opkg: Fix usage with pkgrepo.managed Currently, when using `pkgrepo.managed`, the following error is reported: ``` TypeError: get_repo() got an unexpected keyword argument 'ppa_auth' ``` Fix this by adding `**kwargs` to `get_repo` so that it will absorb unused kwargs instead of erroring out when given an unknown kwarg. Did the same thing for all other public APIs in this file. Signed-off-by: Sergey Kizunov --- salt/modules/opkg.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/salt/modules/opkg.py b/salt/modules/opkg.py index 33449dabf5..137380ea79 100644 --- a/salt/modules/opkg.py +++ b/salt/modules/opkg.py @@ -129,7 +129,7 @@ def version(*names, **kwargs): return __salt__['pkg_resource.version'](*names, **kwargs) -def refresh_db(): +def refresh_db(**kwargs): # pylint: disable=unused-argument ''' Updates the opkg database to latest packages based upon repositories @@ -456,7 +456,7 @@ def purge(name=None, pkgs=None, **kwargs): # pylint: disable=unused-argument return remove(name=name, pkgs=pkgs) -def upgrade(refresh=True): +def upgrade(refresh=True, **kwargs): # pylint: disable=unused-argument ''' Upgrades all packages via ``opkg upgrade`` @@ -739,7 +739,7 @@ def list_pkgs(versions_as_list=False, **kwargs): return ret -def list_upgrades(refresh=True): +def list_upgrades(refresh=True, **kwargs): # pylint: disable=unused-argument ''' List all available package upgrades. @@ -908,7 +908,7 @@ def info_installed(*names, **kwargs): return ret -def upgrade_available(name): +def upgrade_available(name, **kwargs): # pylint: disable=unused-argument ''' Check whether or not an upgrade is available for a given package @@ -921,7 +921,7 @@ def upgrade_available(name): return latest_version(name) != '' -def version_cmp(pkg1, pkg2, ignore_epoch=False): +def version_cmp(pkg1, pkg2, ignore_epoch=False, **kwargs): # pylint: disable=unused-argument ''' Do a cmp-style comparison on two packages. Return -1 if pkg1 < pkg2, 0 if pkg1 == pkg2, and 1 if pkg1 > pkg2. Return None if there was a problem @@ -969,7 +969,7 @@ def version_cmp(pkg1, pkg2, ignore_epoch=False): return None -def list_repos(): +def list_repos(**kwargs): # pylint: disable=unused-argument ''' Lists all repos on /etc/opkg/*.conf @@ -1006,7 +1006,7 @@ def list_repos(): return repos -def get_repo(alias): +def get_repo(alias, **kwargs): # pylint: disable=unused-argument ''' Display a repo from the /etc/opkg/*.conf @@ -1077,7 +1077,7 @@ def _mod_repo_in_file(alias, repostr, filepath): fhandle.writelines(output) -def del_repo(alias): +def del_repo(alias, **kwargs): # pylint: disable=unused-argument ''' Delete a repo from /etc/opkg/*.conf @@ -1191,7 +1191,7 @@ def mod_repo(alias, **kwargs): refresh_db() -def file_list(*packages): +def file_list(*packages, **kwargs): # pylint: disable=unused-argument ''' List the files that belong to a package. Not specifying any packages will return a list of _every_ file on the system's package database (not @@ -1212,7 +1212,7 @@ def file_list(*packages): return {'errors': output['errors'], 'files': files} -def file_dict(*packages): +def file_dict(*packages, **kwargs): # pylint: disable=unused-argument ''' List the files that belong to a package, grouped by package. Not specifying any packages will return a list of _every_ file on the system's @@ -1254,7 +1254,7 @@ def file_dict(*packages): return {'errors': errors, 'packages': ret} -def owner(*paths): +def owner(*paths, **kwargs): # pylint: disable=unused-argument ''' Return the name of the package that owns the file. Multiple file paths can be passed. Like :mod:`pkg.version Date: Thu, 12 Oct 2017 17:39:17 -0500 Subject: [PATCH 037/184] Windows: Fix usage of pkgrepo state `pkgrepo.py` contains this line: ``` from salt.modules.aptpkg import _strip_uri ``` However, `salt.modules.aptpkg` is just not importable on Windows. Instead of trying to make this module importable on Windows (which clearly isn't the intent of the module), placed a copy of `_strip_uri` into `pkgrepo.py` as this is a small function. Signed-off-by: Sergey Kizunov --- salt/states/pkgrepo.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/salt/states/pkgrepo.py b/salt/states/pkgrepo.py index 1d076d1154..f802711a8d 100644 --- a/salt/states/pkgrepo.py +++ b/salt/states/pkgrepo.py @@ -92,7 +92,7 @@ import sys # Import salt libs from salt.exceptions import CommandExecutionError, SaltInvocationError -from salt.modules.aptpkg import _strip_uri +from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin from salt.state import STATE_INTERNAL_KEYWORDS as _STATE_INTERNAL_KEYWORDS import salt.utils import salt.utils.pkg.deb @@ -106,6 +106,18 @@ def __virtual__(): return 'pkg.mod_repo' in __salt__ +def _strip_uri(repo): + ''' + Remove the trailing slash from the URI in a repo definition + ''' + splits = repo.split() + for idx in range(len(splits)): + if any(splits[idx].startswith(x) + for x in ('http://', 'https://', 'ftp://')): + splits[idx] = splits[idx].rstrip('/') + return ' '.join(splits) + + def managed(name, ppa=None, **kwargs): ''' This state manages software package repositories. Currently, :mod:`yum From 344923e231c777d1cfc0ad25e77b59d4ef20fed1 Mon Sep 17 00:00:00 2001 From: Vasili Syrakis Date: Fri, 13 Oct 2017 22:54:58 +1100 Subject: [PATCH 038/184] Catch on empty Virtualbox network addr #43427 --- salt/utils/virtualbox.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/utils/virtualbox.py b/salt/utils/virtualbox.py index c055a6ff8a..6aa108f57d 100644 --- a/salt/utils/virtualbox.py +++ b/salt/utils/virtualbox.py @@ -281,7 +281,10 @@ def vb_get_network_addresses(machine_name=None, machine=None): # We can't trust virtualbox to give us up to date guest properties if the machine isn't running # For some reason it may give us outdated (cached?) values if machine.state == _virtualboxManager.constants.MachineState_Running: - total_slots = int(machine.getGuestPropertyValue('/VirtualBox/GuestInfo/Net/Count')) + try: + total_slots = int(machine.getGuestPropertyValue('/VirtualBox/GuestInfo/Net/Count')) + except ValueError: + total_slots = 0 for i in range(total_slots): try: address = machine.getGuestPropertyValue('/VirtualBox/GuestInfo/Net/{0}/V4/IP'.format(i)) From f749cafa25c95fcd2f1579a1f8e746741cb9517c Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Fri, 13 Oct 2017 09:54:07 -0600 Subject: [PATCH 039/184] don't filter if return is not a dict --- salt/modules/state.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/state.py b/salt/modules/state.py index e05e41983f..c42d874ee1 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -863,8 +863,8 @@ def highstate(test=None, queue=False, **kwargs): finally: st_.pop_active() - if __salt__['config.option']('state_data', '') == 'terse' or \ - kwargs.get('terse'): + if isinstance(ret, dict) and (__salt__['config.option']('state_data', '') == 'terse' or \ + kwargs.get('terse')): ret = _filter_running(ret) serial = salt.payload.Serial(__opts__) From c1a3b048a490f458a34a156e82b8f1fe2069c8ae Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 10 Oct 2017 08:44:57 -0700 Subject: [PATCH 040/184] Adding the ability to pass pillar values in when using state.sls_id --- salt/modules/state.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/salt/modules/state.py b/salt/modules/state.py index e5616af7d7..484bdd0e82 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -1377,6 +1377,21 @@ def sls_id(id_, mods, test=None, queue=False, **kwargs): :conf_minion:`pillarenv` minion config option nor this CLI argument is used, all Pillar environments will be merged together. + pillar + Custom Pillar values, passed as a dictionary of key-value pairs + + .. code-block:: bash + + salt '*' state.sls_id my_state my_module pillar='{"foo": "bar"}' + + .. note:: + Values passed this way will override Pillar values set via + ``pillar_roots`` or an external Pillar source. + + .. versionchanged:: Oxygen + GPG-encrypted CLI Pillar data is now supported via the GPG + renderer. See :ref:`here ` for details. + CLI Example: .. code-block:: bash @@ -1397,12 +1412,26 @@ def sls_id(id_, mods, test=None, queue=False, **kwargs): if opts['environment'] is None: opts['environment'] = 'base' + pillar_override = kwargs.get('pillar') + pillar_enc = kwargs.get('pillar_enc') + if pillar_enc is None \ + and pillar_override is not None \ + and not isinstance(pillar_override, dict): + raise SaltInvocationError( + 'Pillar data must be formatted as a dictionary, unless pillar_enc ' + 'is specified.' + ) + try: st_ = salt.state.HighState(opts, + pillar_override, + pillar_enc=pillar_enc, proxy=__proxy__, initial_pillar=_get_initial_pillar(opts)) except NameError: st_ = salt.state.HighState(opts, + pillar_override, + pillar_enc=pillar_enc, initial_pillar=_get_initial_pillar(opts)) if not _check_pillar(kwargs, st_.opts['pillar']): From cc962ba400ccd775e48efd3d780bfc2c0df7b697 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 10 Oct 2017 09:02:10 -0700 Subject: [PATCH 041/184] Updating docstring with right versionadded information --- salt/modules/state.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/salt/modules/state.py b/salt/modules/state.py index 484bdd0e82..85f6ad4189 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -1388,9 +1388,7 @@ def sls_id(id_, mods, test=None, queue=False, **kwargs): Values passed this way will override Pillar values set via ``pillar_roots`` or an external Pillar source. - .. versionchanged:: Oxygen - GPG-encrypted CLI Pillar data is now supported via the GPG - renderer. See :ref:`here ` for details. + .. versionadded:: Oxygen CLI Example: From b8a2cd754419fdae1e9b4a33fec2073f3dfbf293 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 13 Oct 2017 09:15:33 -0700 Subject: [PATCH 042/184] Clarifying documention for pillar kwarg for sls and sls_id. --- salt/modules/state.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/salt/modules/state.py b/salt/modules/state.py index 85f6ad4189..ba514e0e1e 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -921,8 +921,9 @@ def sls(mods, test=None, exclude=None, queue=False, **kwargs): salt '*' state.apply test pillar='{"foo": "bar"}' .. note:: - Values passed this way will override Pillar values set via - ``pillar_roots`` or an external Pillar source. + Values passed this way will override existing Pillar values set via + ``pillar_roots`` or an external Pillar source. Pillar values that + are not included in the kwarg will not be overwritten. .. versionchanged:: 2016.3.0 GPG-encrypted CLI Pillar data is now supported via the GPG @@ -1385,8 +1386,9 @@ def sls_id(id_, mods, test=None, queue=False, **kwargs): salt '*' state.sls_id my_state my_module pillar='{"foo": "bar"}' .. note:: - Values passed this way will override Pillar values set via - ``pillar_roots`` or an external Pillar source. + Values passed this way will override existing Pillar values set via + ``pillar_roots`` or an external Pillar source. Pillar values that + are not included in the kwarg will not be overwritten. .. versionadded:: Oxygen From 5455c5053b5043660cd12a4bb833dde78b799c76 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Fri, 13 Oct 2017 14:03:16 -0600 Subject: [PATCH 043/184] fix pylint --- salt/modules/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/state.py b/salt/modules/state.py index c42d874ee1..14156fe5f1 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -863,7 +863,7 @@ def highstate(test=None, queue=False, **kwargs): finally: st_.pop_active() - if isinstance(ret, dict) and (__salt__['config.option']('state_data', '') == 'terse' or \ + if isinstance(ret, dict) and (__salt__['config.option']('state_data', '') == 'terse' or kwargs.get('terse')): ret = _filter_running(ret) From bd2490b1498d91b1a560e04cad3f157d8916f6d6 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Fri, 13 Oct 2017 15:10:08 -0600 Subject: [PATCH 044/184] OpenNebula does not require the template_id to be specified --- salt/cloud/clouds/opennebula.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/cloud/clouds/opennebula.py b/salt/cloud/clouds/opennebula.py index 6c2ff5c670..b3280fd6fe 100644 --- a/salt/cloud/clouds/opennebula.py +++ b/salt/cloud/clouds/opennebula.py @@ -4468,7 +4468,8 @@ def _list_nodes(full=False): pass vms[name]['id'] = vm.find('ID').text - vms[name]['image'] = vm.find('TEMPLATE').find('TEMPLATE_ID').text + if vm.find('TEMPLATE').find('TEMPLATE_ID'): + vms[name]['image'] = vm.find('TEMPLATE').find('TEMPLATE_ID').text vms[name]['name'] = name vms[name]['size'] = {'cpu': cpu_size, 'memory': memory_size} vms[name]['state'] = vm.find('STATE').text From 881f1822f28882c1087a56d73c54325f33c896cc Mon Sep 17 00:00:00 2001 From: Roald Nefs Date: Sat, 14 Oct 2017 20:05:30 +0200 Subject: [PATCH 045/184] Format fix code example local returner doc --- salt/returners/local.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/returners/local.py b/salt/returners/local.py index 703b16982c..5844011b23 100644 --- a/salt/returners/local.py +++ b/salt/returners/local.py @@ -3,7 +3,9 @@ The local returner is used to test the returner interface, it just prints the return data to the console to verify that it is being passed properly - To use the local returner, append '--return local' to the salt command. ex: +To use the local returner, append '--return local' to the salt command. ex: + +.. code-block:: bash salt '*' test.ping --return local ''' From 16e1c1dfc86920b7a00dbf7c39b805c359e4d13b Mon Sep 17 00:00:00 2001 From: Matthew Summers Date: Mon, 16 Oct 2017 09:47:40 -0500 Subject: [PATCH 046/184] fixed test addressing issue #43307, disk.format_ to disk.format --- tests/unit/states/test_blockdev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/states/test_blockdev.py b/tests/unit/states/test_blockdev.py index e5899f1c70..9b559dddfe 100644 --- a/tests/unit/states/test_blockdev.py +++ b/tests/unit/states/test_blockdev.py @@ -100,7 +100,7 @@ class BlockdevTestCase(TestCase, LoaderModuleMockMixin): # Test state return when block device format fails with patch.dict(blockdev.__salt__, {'cmd.run': MagicMock(return_value=mock_ext4), - 'disk.format_': MagicMock(return_value=True)}): + 'disk.format': MagicMock(return_value=True)}): comt = ('Failed to format {0}'.format(name)) ret.update({'comment': comt, 'result': False}) with patch.object(salt.utils, 'which', From 37c7980880388d13052001f4e031b04555824361 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 16 Oct 2017 10:03:02 -0600 Subject: [PATCH 047/184] Add note about GPG signing to PR template --- .github/PULL_REQUEST_TEMPLATE.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 642eabb9eb..b75679cd9a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -12,4 +12,10 @@ Remove this section if not relevant Yes/No +### Commits signed with GPG? + +Yes/No + Please review [Salt's Contributing Guide](https://docs.saltstack.com/en/latest/topics/development/contributing.html) for best practices. + +See GitHub's [page on GPG signing](https://help.github.com/articles/signing-commits-using-gpg/) for more information signing commits with GPG. From a4e2d8059d9f957b854dadfafe5d8cf5eba52391 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 4 Oct 2017 15:10:58 -0600 Subject: [PATCH 048/184] Fix `unit.templates.test_jinja` for Windows Fix problem with files that end with new line on Windows Fix some problems with regex Fix some unicode conversion issues --- salt/utils/templates.py | 9 +++++++-- tests/unit/templates/test_jinja.py | 20 ++++++++++---------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/salt/utils/templates.py b/salt/utils/templates.py index b4bf049dc1..e6da2c119e 100644 --- a/salt/utils/templates.py +++ b/salt/utils/templates.py @@ -168,8 +168,13 @@ def wrap_tmpl_func(render_str): if six.PY2: output = output.encode(SLS_ENCODING) if salt.utils.is_windows(): + newline = False + if output.endswith(('\n', os.linesep)): + newline = True # Write out with Windows newlines output = os.linesep.join(output.splitlines()) + if newline: + output += os.linesep except SaltRenderError as exc: log.error("Rendering exception occurred: {0}".format(exc)) @@ -293,7 +298,7 @@ def render_jinja_tmpl(tmplstr, context, tmplpath=None): # http://jinja.pocoo.org/docs/api/#unicode tmplstr = tmplstr.decode(SLS_ENCODING) - if tmplstr.endswith('\n'): + if tmplstr.endswith(os.linesep): newline = True if not saltenv: @@ -462,7 +467,7 @@ def render_jinja_tmpl(tmplstr, context, tmplpath=None): # Workaround a bug in Jinja that removes the final newline # (https://github.com/mitsuhiko/jinja2/issues/75) if newline: - output += '\n' + output += os.linesep return output diff --git a/tests/unit/templates/test_jinja.py b/tests/unit/templates/test_jinja.py index 7a96608825..4ad4b618f8 100644 --- a/tests/unit/templates/test_jinja.py +++ b/tests/unit/templates/test_jinja.py @@ -177,7 +177,7 @@ class TestGetTemplate(TestCase): out = render_jinja_tmpl( fp_.read(), dict(opts=self.local_opts, saltenv='test', salt=self.local_salt)) - self.assertEqual(out, 'world\n') + self.assertEqual(out, 'world' + os.linesep) def test_fallback_noloader(self): ''' @@ -189,7 +189,7 @@ class TestGetTemplate(TestCase): out = render_jinja_tmpl( fp_.read(), dict(opts=self.local_opts, saltenv='test', salt=self.local_salt)) - self.assertEqual(out, 'Hey world !a b !\n') + self.assertEqual(out, 'Hey world !a b !' + os.linesep) def test_saltenv(self): ''' @@ -208,7 +208,7 @@ class TestGetTemplate(TestCase): 'file_roots': self.local_opts['file_roots'], 'pillar_roots': self.local_opts['pillar_roots']}, a='Hi', b='Salt', saltenv='test', salt=self.local_salt)) - self.assertEqual(out, 'Hey world !Hi Salt !\n') + self.assertEqual(out, 'Hey world !Hi Salt !' + os.linesep) self.assertEqual(fc.requests[0]['path'], 'salt://macro') def test_macro_additional_log_for_generalexc(self): @@ -217,7 +217,7 @@ class TestGetTemplate(TestCase): more output from trace. ''' expected = r'''Jinja error:.*division.* -.*/macrogeneral\(2\): +.*macrogeneral\(2\): --- \{% macro mymacro\(\) -%\} \{\{ 1/0 \}\} <====================== @@ -241,7 +241,7 @@ class TestGetTemplate(TestCase): more output from trace. ''' expected = r'''Jinja variable 'b' is undefined -.*/macroundefined\(2\): +.*macroundefined\(2\): --- \{% macro mymacro\(\) -%\} \{\{b.greetee\}\} <-- error is here <====================== @@ -264,7 +264,7 @@ class TestGetTemplate(TestCase): If we failed in a macro, get more output from trace. ''' expected = r'''Jinja syntax error: expected token .*end.*got '-'.* -.*/macroerror\(2\): +.*macroerror\(2\): --- # macro \{% macro mymacro\(greeting, greetee='world'\) -\} <-- error is here <====================== @@ -294,7 +294,7 @@ class TestGetTemplate(TestCase): 'file_roots': self.local_opts['file_roots'], 'pillar_roots': self.local_opts['pillar_roots']}, a='Hi', b='Sàlt', saltenv='test', salt=self.local_salt)) - self.assertEqual(out, u'Hey world !Hi Sàlt !\n') + self.assertEqual(out, salt.utils.to_unicode('Hey world !Hi Sàlt !' + os.linesep)) self.assertEqual(fc.requests[0]['path'], 'salt://macro') filename = os.path.join(TEMPLATES_DIR, 'files', 'test', 'non_ascii') @@ -305,7 +305,7 @@ class TestGetTemplate(TestCase): 'file_roots': self.local_opts['file_roots'], 'pillar_roots': self.local_opts['pillar_roots']}, a='Hi', b='Sàlt', saltenv='test', salt=self.local_salt)) - self.assertEqual(u'Assunção\n', out) + self.assertEqual(u'Assunção' + os.linesep, out) self.assertEqual(fc.requests[0]['path'], 'salt://macro') @skipIf(HAS_TIMELIB is False, 'The `timelib` library is not installed.') @@ -340,8 +340,8 @@ class TestGetTemplate(TestCase): with salt.utils.fopen(out['data']) as fp: result = fp.read() if six.PY2: - result = result.decode('utf-8') - self.assertEqual(u'Assunção\n', result) + result = salt.utils.to_unicode(result) + self.assertEqual(salt.utils.to_unicode('Assunção' + os.linesep), result) def test_get_context_has_enough_context(self): template = '1\n2\n3\n4\n5\n6\n7\n8\n9\na\nb\nc\nd\ne\nf' From 7fe23503657bb02586fb6f2501d16f2f782c7c7f Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 20 Sep 2017 15:38:54 -0600 Subject: [PATCH 049/184] Add support for multimaster setup Allows you to pass a comma-delimited list of masters to the master combo box or the /master command line switch Parses the existing minion config file to get multimaster settings Adds an checkbox to use the existing config in the minion config page When the box is checked, show the existing config grayed out When unchecked, show default values Adss the /use-existing-config= command line switch for use from the command line Adds support for a help switch on the command line (/?) to display the supported command line paramaters --- pkg/windows/installer/Salt-Minion-Setup.nsi | 355 ++++++++++++++++++-- 1 file changed, 334 insertions(+), 21 deletions(-) diff --git a/pkg/windows/installer/Salt-Minion-Setup.nsi b/pkg/windows/installer/Salt-Minion-Setup.nsi index a8efca2101..783ac57147 100644 --- a/pkg/windows/installer/Salt-Minion-Setup.nsi +++ b/pkg/windows/installer/Salt-Minion-Setup.nsi @@ -11,6 +11,7 @@ !define PRODUCT_UNINST_KEY "Software\Microsoft\Windows\CurrentVersion\Uninstall\${PRODUCT_NAME}" !define PRODUCT_UNINST_KEY_OTHER "Software\Microsoft\Windows\CurrentVersion\Uninstall\${PRODUCT_NAME_OTHER}" !define PRODUCT_UNINST_ROOT_KEY "HKLM" +!define OUTFILE "Salt-Minion-${PRODUCT_VERSION}-Py${PYTHON_VERSION}-${CPUARCH}-Setup.exe" # Import Libraries !include "MUI2.nsh" @@ -52,6 +53,15 @@ ${StrStrAdv} Pop "${ResultVar}" !macroend +# Part of the Explode function for Strings +!define Explode "!insertmacro Explode" +!macro Explode Length Separator String + Push `${Separator}` + Push `${String}` + Call Explode + Pop `${Length}` +!macroend + ############################################################################### # Configure Pages, Ordering, and Configuration @@ -92,10 +102,17 @@ Var Dialog Var Label Var CheckBox_Minion_Start Var CheckBox_Minion_Start_Delayed +Var ConfigMasterHost Var MasterHost Var MasterHost_State +Var ConfigMinionName Var MinionName Var MinionName_State +Var ExistingConfigFound +Var UseExistingConfig +Var UseExistingConfig_State +Var WarningExistingConfig +Var WarningDefaultConfig Var StartMinion Var StartMinionDelayed Var DeleteInstallDir @@ -115,27 +132,105 @@ Function pageMinionConfig Abort ${EndIf} + # Master IP or Hostname Dialog Control ${NSD_CreateLabel} 0 0 100% 12u "Master IP or Hostname:" Pop $Label ${NSD_CreateText} 0 13u 100% 12u $MasterHost_State Pop $MasterHost + # Minion ID Dialog Control ${NSD_CreateLabel} 0 30u 100% 12u "Minion Name:" Pop $Label ${NSD_CreateText} 0 43u 100% 12u $MinionName_State Pop $MinionName + # Use Existing Config Checkbox + ${NSD_CreateCheckBox} 0 65u 100% 12u "&Use Existing Config" + Pop $UseExistingConfig + ${NSD_OnClick} $UseExistingConfig pageMinionConfig_OnClick + + # Add Existing Config Warning Label + ${NSD_CreateLabel} 0 80u 100% 60u "The values above are taken from an \ + existing configuration found in `c:\salt\conf\minion`. Configuration \ + settings defined in the `minion.d` directories, if they exist, are not \ + shown here.$\r$\n\ + $\r$\n\ + Clicking `Install` will leave the existing config unchanged." + Pop $WarningExistingConfig + CreateFont $0 "Arial" 10 500 /ITALIC + SendMessage $WarningExistingConfig ${WM_SETFONT} $0 1 + SetCtlColors $WarningExistingConfig 0xBB0000 transparent + + # Add Default Config Warning Label + ${NSD_CreateLabel} 0 80u 100% 60u "Clicking `Install` will remove the \ + the existing minion config file and remove the minion.d directories. \ + The values above will be used in the new default config." + Pop $WarningDefaultConfig + CreateFont $0 "Arial" 10 500 /ITALIC + SendMessage $WarningDefaultConfig ${WM_SETFONT} $0 1 + SetCtlColors $WarningDefaultConfig 0xBB0000 transparent + + # If no existing config found, disable the checkbox and stuff + # Set UseExistingConfig_State to 0 + ${If} $ExistingConfigFound == 0 + StrCpy $UseExistingConfig_State 0 + ShowWindow $UseExistingConfig ${SW_HIDE} + ShowWindow $WarningExistingConfig ${SW_HIDE} + ShowWindow $WarningDefaultConfig ${SW_HIDE} + ${Endif} + + ${NSD_SetState} $UseExistingConfig $UseExistingConfig_State + + Call pageMinionConfig_OnClick + nsDialogs::Show FunctionEnd +Function pageMinionConfig_OnClick + + # You have to pop the top handle to keep the stack clean + Pop $R0 + + # Assign the current checkbox state to the variable + ${NSD_GetState} $UseExistingConfig $UseExistingConfig_State + + # Validate the checkboxes + ${If} $UseExistingConfig_State == ${BST_CHECKED} + # Use Existing Config is checked, show warning + ShowWindow $WarningExistingConfig ${SW_SHOW} + EnableWindow $MasterHost 0 + EnableWindow $MinionName 0 + ${NSD_SetText} $MasterHost $ConfigMasterHost + ${NSD_SetText} $MinionName $ConfigMinionName + ${If} $ExistingConfigFound == 1 + ShowWindow $WarningDefaultConfig ${SW_HIDE} + ${Endif} + ${Else} + # Use Existing Config is not checked, hide the warning + ShowWindow $WarningExistingConfig ${SW_HIDE} + EnableWindow $MasterHost 1 + EnableWindow $MinionName 1 + ${NSD_SetText} $MasterHost $MasterHost_State + ${NSD_SetText} $MinionName $MinionName_State + ${If} $ExistingConfigFound == 1 + ShowWindow $WarningDefaultConfig ${SW_SHOW} + ${Endif} + ${EndIf} + +FunctionEnd + + Function pageMinionConfig_Leave ${NSD_GetText} $MasterHost $MasterHost_State ${NSD_GetText} $MinionName $MinionName_State + ${NSD_GetState} $UseExistingConfig $UseExistingConfig_State + + Call RemoveExistingConfig FunctionEnd @@ -194,7 +289,7 @@ FunctionEnd !else Name "${PRODUCT_NAME} ${PRODUCT_VERSION}" !endif -OutFile "Salt-Minion-${PRODUCT_VERSION}-Py${PYTHON_VERSION}-${CPUARCH}-Setup.exe" +OutFile "${OutFile}" InstallDir "c:\salt" InstallDirRegKey HKLM "${PRODUCT_DIR_REGKEY}" "" ShowInstDetails show @@ -311,8 +406,6 @@ SectionEnd Function .onInit - Call getMinionConfig - Call parseCommandLineSwitches # Check for existing installation @@ -364,6 +457,23 @@ Function .onInit skipUninstall: + Call getMinionConfig + + IfSilent 0 +2 + Call RemoveExistingConfig + +FunctionEnd + + +Function RemoveExistingConfig + + ${If} $ExistingConfigFound == 1 + ${AndIf} $UseExistingConfig_State == 0 + # Wipe out the Existing Config + Delete "$INSTDIR\conf\minion" + RMDir /r "$INSTDIR\conf\minion.d" + ${EndIf} + FunctionEnd @@ -407,7 +517,9 @@ Section -Post nsExec::Exec "nssm.exe set salt-minion AppStopMethodConsole 24000" nsExec::Exec "nssm.exe set salt-minion AppStopMethodWindow 2000" - Call updateMinionConfig + ${If} $UseExistingConfig_State == 0 + Call updateMinionConfig + ${EndIf} Push "C:\salt" Call AddToPath @@ -534,18 +646,28 @@ FunctionEnd # Helper Functions ############################################################################### Function MsiQueryProductState + # Used for detecting VCRedist Installation + !define INSTALLSTATE_DEFAULT "5" - !define INSTALLSTATE_DEFAULT "5" - - Pop $R0 - StrCpy $NeedVcRedist "False" - System::Call "msi::MsiQueryProductStateA(t '$R0') i.r0" - StrCmp $0 ${INSTALLSTATE_DEFAULT} +2 0 - StrCpy $NeedVcRedist "True" + Pop $R0 + StrCpy $NeedVcRedist "False" + System::Call "msi::MsiQueryProductStateA(t '$R0') i.r0" + StrCmp $0 ${INSTALLSTATE_DEFAULT} +2 0 + StrCpy $NeedVcRedist "True" FunctionEnd +#------------------------------------------------------------------------------ +# Trim Function +# - Trim whitespace from the beginning and end of a string +# - Trims spaces, \r, \n, \t +# +# Usage: +# Push " some string " +# Call Trim +# Pop $0 ; "some string" +#------------------------------------------------------------------------------ Function Trim Exch $R1 # Original string @@ -580,6 +702,79 @@ Function Trim FunctionEnd +Function Explode + # Initialize variables + Var /GLOBAL explString + Var /GLOBAL explSeparator + Var /GLOBAL explStrLen + Var /GLOBAL explSepLen + Var /GLOBAL explOffset + Var /GLOBAL explTmp + Var /GLOBAL explTmp2 + Var /GLOBAL explTmp3 + Var /GLOBAL explArrCount + + # Get input from user + Pop $explString + Pop $explSeparator + + # Calculates initial values + StrLen $explStrLen $explString + StrLen $explSepLen $explSeparator + StrCpy $explArrCount 1 + + ${If} $explStrLen <= 1 # If we got a single character + ${OrIf} $explSepLen > $explStrLen # or separator is larger than the string, + Push $explString # then we return initial string with no change + Push 1 # and set array's length to 1 + Return + ${EndIf} + + # Set offset to the last symbol of the string + StrCpy $explOffset $explStrLen + IntOp $explOffset $explOffset - 1 + + # Clear temp string to exclude the possibility of appearance of occasional data + StrCpy $explTmp "" + StrCpy $explTmp2 "" + StrCpy $explTmp3 "" + + # Loop until the offset becomes negative + ${Do} + # If offset becomes negative, it is time to leave the function + ${IfThen} $explOffset == -1 ${|} ${ExitDo} ${|} + + # Remove everything before and after the searched part ("TempStr") + StrCpy $explTmp $explString $explSepLen $explOffset + + ${If} $explTmp == $explSeparator + # Calculating offset to start copy from + IntOp $explTmp2 $explOffset + $explSepLen # Offset equals to the current offset plus length of separator + StrCpy $explTmp3 $explString "" $explTmp2 + + Push $explTmp3 # Throwing array item to the stack + IntOp $explArrCount $explArrCount + 1 # Increasing array's counter + + StrCpy $explString $explString $explOffset 0 # Cutting all characters beginning with the separator entry + StrLen $explStrLen $explString + ${EndIf} + + ${If} $explOffset = 0 # If the beginning of the line met and there is no separator, + # copying the rest of the string + ${If} $explSeparator == "" # Fix for the empty separator + IntOp $explArrCount $explArrCount - 1 + ${Else} + Push $explString + ${EndIf} + ${EndIf} + + IntOp $explOffset $explOffset - 1 + ${Loop} + + Push $explArrCount +FunctionEnd + + #------------------------------------------------------------------------------ # StrStr Function # - find substring in a string @@ -816,6 +1011,9 @@ FunctionEnd ############################################################################### Function getMinionConfig + # Set Config Found Default Value + StrCpy $ExistingConfigFound 0 + confFind: IfFileExists "$INSTDIR\conf\minion" confFound confNotFound @@ -828,24 +1026,42 @@ Function getMinionConfig ${EndIf} confFound: + StrCpy $ExistingConfigFound 1 FileOpen $0 "$INSTDIR\conf\minion" r - ClearErrors confLoop: - FileRead $0 $1 - IfErrors EndOfFile - ${StrLoc} $2 $1 "master:" ">" - ${If} $2 == 0 - ${StrStrAdv} $2 $1 "master: " ">" ">" "0" "0" "0" - ${Trim} $2 $2 - StrCpy $MasterHost_State $2 + ClearErrors # Clear Errors + FileRead $0 $1 # Read the next line + IfErrors EndOfFile # Error is probably EOF + ${StrLoc} $2 $1 "master:" ">" # Find `master:` starting at the beginning + ${If} $2 == 0 # If it found it in the first position, then it is defined + ${StrStrAdv} $2 $1 "master: " ">" ">" "0" "0" "0" # Read everything after `master: ` + ${Trim} $2 $2 # Trim white space + ${If} $2 == "" # If it's empty, it's probably a list + masterLoop: + ClearErrors # Clear Errors + FileRead $0 $1 # Read the next line + IfErrors EndOfFile # Error is probably EOF + ${StrStrAdv} $2 $1 "- " ">" ">" "0" "0" "0" # Read everything after `- ` + ${Trim} $2 $2 # Trim white space + ${IfNot} $2 == "" # If it's not empty, we found something + ${If} $ConfigMasterHost == "" # Is the default `salt` there + StrCpy $ConfigMasterHost $2 # If so, make the first item the new entry + ${Else} + StrCpy $ConfigMasterHost "$ConfigMasterHost,$2" # Append the new master, comma separated + ${EndIf} + Goto masterLoop # Check the next one + ${EndIf} + ${Else} + StrCpy $ConfigMasterHost $2 # A single master entry ${EndIf} + ${EndIf} ${StrLoc} $2 $1 "id:" ">" ${If} $2 == 0 ${StrStrAdv} $2 $1 "id: " ">" ">" "0" "0" "0" ${Trim} $2 $2 - StrCpy $MinionName_State $2 + StrCpy $ConfigMinionName $2 ${EndIf} Goto confLoop @@ -855,6 +1071,14 @@ Function getMinionConfig confReallyNotFound: + # Set Default Config Values if not found + ${If} $ConfigMasterHost == "" + StrCpy $ConfigMasterHost "salt" + ${EndIf} + ${If} $ConfigMinionName == "" + StrCpy $ConfigMinionName "hostname" + ${EndIf} + FunctionEnd @@ -869,12 +1093,28 @@ Function updateMinionConfig FileRead $0 $2 # read line from target file IfErrors done # end if errors are encountered (end of line) + ${If} $MasterHost_State != "" # if master is empty ${AndIf} $MasterHost_State != "salt" # and if master is not 'salt' ${StrLoc} $3 $2 "master:" ">" # where is 'master:' in this line ${If} $3 == 0 # is it in the first... ${OrIf} $3 == 1 # or second position (account for comments) - StrCpy $2 "master: $MasterHost_State$\r$\n" # write the master + + ${Explode} $9 "," $MasterHost_state + ${If} $9 == 1 + StrCpy $2 "master: $MasterHost_State$\r$\n" # write the master + ${Else} + StrCpy $2 "master:" + + loop_explode: + pop $8 + ${Trim} $8 $8 + StrCpy $2 "$2$\r$\n - $8" + IntOp $9 $9 - 1 + ${If} $9 >= 1 + Goto loop_explode + ${EndIf} + ${EndIf} # close if statement ${EndIf} # close if statement ${EndIf} # close if statement @@ -905,6 +1145,67 @@ Function parseCommandLineSwitches # Load the parameters ${GetParameters} $R0 + # Display Help + ClearErrors + ${GetOptions} $R0 "/?" $R1 + IfErrors display_help_not_found + + System::Call 'kernel32::GetStdHandle(i -11)i.r0' + System::Call 'kernel32::AttachConsole(i -1)i.r1' + ${If} $0 = 0 + ${OrIf} $1 = 0 + System::Call 'kernel32::AllocConsole()' + System::Call 'kernel32::GetStdHandle(i -11)i.r0' + ${EndIf} + FileWrite $0 "$\n" + FileWrite $0 "$\n" + FileWrite $0 "Help for Salt Minion installation$\n" + FileWrite $0 "===============================================================================$\n" + FileWrite $0 "$\n" + FileWrite $0 "/minion-name=$\t$\tA string value to set the minion name. Default is$\n" + FileWrite $0 "$\t$\t$\t'hostname'. Setting the minion name will replace$\n" + FileWrite $0 "$\t$\t$\texisting config with a default config. Cannot be$\n" + FileWrite $0 "$\t$\t$\tused in conjunction with /use-existing-config=1$\n" + FileWrite $0 "$\n" + FileWrite $0 "/master=$\t$\tA string value to set the IP address or hostname of$\n" + FileWrite $0 "$\t$\t$\tthe master. Default value is 'salt'. You may pass a$\n" + FileWrite $0 "$\t$\t$\tsingle master, or a comma separated list of masters.$\n" + FileWrite $0 "$\t$\t$\tSetting the master will replace existing config with$\n" + FileWrite $0 "$\t$\t$\ta default config. Cannot be used in conjunction with$\n" + FileWrite $0 "$\t$\t$\t/use-existing-config=1$\n" + FileWrite $0 "$\n" + FileWrite $0 "/start-minion=$\t$\t1 will start the service, 0 will not. Default is 1$\n" + FileWrite $0 "$\n" + FileWrite $0 "/start-minion-delayed$\tSet the minion start type to 'Automatic (Delayed Start)'$\n" + FileWrite $0 "$\n" + FileWrite $0 "/use-existing-config=$\t1 will use the existing config if present, 0 will$\n" + FileWrite $0 "$\t$\t$\treplace existing config with a default config. Default$\n" + FileWrite $0 "$\t$\t$\tis 1. If this is set to 1, values passed in$\n" + FileWrite $0 "$\t$\t$\t/minion-name and /master will be ignored$\n" + FileWrite $0 "$\n" + FileWrite $0 "/S$\t$\t$\tInstall Salt silently$\n" + FileWrite $0 "$\n" + FileWrite $0 "/?$\t$\t$\tDisplay this help screen$\n" + FileWrite $0 "$\n" + FileWrite $0 "-------------------------------------------------------------------------------$\n" + FileWrite $0 "$\n" + FileWrite $0 "Examples:$\n" + FileWrite $0 "$\n" + FileWrite $0 "${OutFile} /S$\n" + FileWrite $0 "$\n" + FileWrite $0 "${OutFile} /S /minion-name=myminion /master=master.mydomain.com /start-minion-delayed$\n" + FileWrite $0 "$\n" + FileWrite $0 "===============================================================================$\n" + FileWrite $0 "Press Enter to continue..." + System::Free $0 + System::Free $1 + System::Call 'kernel32::FreeConsole()' + Abort + display_help_not_found: + + # Set default value for Use Existing Config + StrCpy $UseExistingConfig_State 1 + # Check for start-minion switches # /start-service is to be deprecated, so we must check for both ${GetOptions} $R0 "/start-service=" $R1 @@ -930,19 +1231,31 @@ Function parseCommandLineSwitches start_minion_delayed_not_found: # Minion Config: Master IP/Name + # If setting master, we don't want to use existing config ${GetOptions} $R0 "/master=" $R1 ${IfNot} $R1 == "" StrCpy $MasterHost_State $R1 + StrCpy $UseExistingConfig_State 0 ${ElseIf} $MasterHost_State == "" StrCpy $MasterHost_State "salt" ${EndIf} # Minion Config: Minion ID + # If setting minion id, we don't want to use existing config ${GetOptions} $R0 "/minion-name=" $R1 ${IfNot} $R1 == "" StrCpy $MinionName_State $R1 + StrCpy $UseExistingConfig_State 0 ${ElseIf} $MinionName_State == "" StrCpy $MinionName_State "hostname" ${EndIf} + # Use Existing Config + # Overrides above settings with user passed settings + ${GetOptions} $R0 "/use-existing-config=" $R1 + ${IfNot} $R1 == "" + # Use Existing Config was passed something, set it + StrCpy $UseExistingConfig_State $R1 + ${EndIf} + FunctionEnd From e9075725dc0a7f7f4a95c92984b404c5fc4580cc Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 21 Sep 2017 14:11:21 -0600 Subject: [PATCH 050/184] Add documenting comments --- pkg/windows/installer/Salt-Minion-Setup.nsi | 49 ++++++++++++++------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/pkg/windows/installer/Salt-Minion-Setup.nsi b/pkg/windows/installer/Salt-Minion-Setup.nsi index 783ac57147..6283d057a7 100644 --- a/pkg/windows/installer/Salt-Minion-Setup.nsi +++ b/pkg/windows/installer/Salt-Minion-Setup.nsi @@ -664,9 +664,13 @@ FunctionEnd # - Trims spaces, \r, \n, \t # # Usage: -# Push " some string " +# Push " some string " ; String to Trim # Call Trim -# Pop $0 ; "some string" +# Pop $0 ; Trimmed String: "some string" +# +# or +# +# ${Trim} $0 $1 ; Trimmed String, String to Trim #------------------------------------------------------------------------------ Function Trim @@ -702,6 +706,22 @@ Function Trim FunctionEnd +#------------------------------------------------------------------------------ +# Explode Function +# - Splits a string based off the passed separator +# - Each item in the string is pushed to the stack +# - The last item pushed to the stack is the length of the array +# +# Usage: +# Push "," ; Separator +# Push "string,to,separate" ; String to explode +# Call Explode +# Pop $0 ; Number of items in the array +# +# or +# +# ${Explode} $0 $1 $2 ; Length, Separator, String +#------------------------------------------------------------------------------ Function Explode # Initialize variables Var /GLOBAL explString @@ -1093,27 +1113,26 @@ Function updateMinionConfig FileRead $0 $2 # read line from target file IfErrors done # end if errors are encountered (end of line) - ${If} $MasterHost_State != "" # if master is empty ${AndIf} $MasterHost_State != "salt" # and if master is not 'salt' ${StrLoc} $3 $2 "master:" ">" # where is 'master:' in this line ${If} $3 == 0 # is it in the first... ${OrIf} $3 == 1 # or second position (account for comments) - ${Explode} $9 "," $MasterHost_state - ${If} $9 == 1 + ${Explode} $9 "," $MasterHost_state # Split the hostname on commas, $9 is the number of items found + ${If} $9 == 1 # 1 means only a single master was passed StrCpy $2 "master: $MasterHost_State$\r$\n" # write the master - ${Else} - StrCpy $2 "master:" + ${Else} # Make a multi-master entry + StrCpy $2 "master:" # Make the first line "master:" - loop_explode: - pop $8 - ${Trim} $8 $8 - StrCpy $2 "$2$\r$\n - $8" - IntOp $9 $9 - 1 - ${If} $9 >= 1 - Goto loop_explode - ${EndIf} + loop_explode: # Start a loop to go through the list in the config + pop $8 # Pop the next item off the stack + ${Trim} $8 $8 # Trim any whitespace + StrCpy $2 "$2$\r$\n - $8" # Add it to the master variable ($2) + IntOp $9 $9 - 1 # Decrement the list count + ${If} $9 >= 1 # If it's not 0 + Goto loop_explode # Do it again + ${EndIf} # close if statement ${EndIf} # close if statement ${EndIf} # close if statement ${EndIf} # close if statement From 7bcf8b48ecaa75b3da2aa13efbc871e2f81cbb40 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 21 Sep 2017 15:16:10 -0600 Subject: [PATCH 051/184] Update documentation with new changes to installer --- doc/topics/installation/windows.rst | 49 ++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/doc/topics/installation/windows.rst b/doc/topics/installation/windows.rst index 06219e73e6..68ded4a510 100644 --- a/doc/topics/installation/windows.rst +++ b/doc/topics/installation/windows.rst @@ -45,11 +45,27 @@ but leave any existing config, cache, and PKI information. Salt Minion Installation ======================== +If the system is missing the appropriate version of the Visual C++ +Redistributable (vcredist) the user will be prompted to install it. Click ``OK`` +to install the vcredist. Click ``Cancel`` to abort the installation without +making modifications to the systeml. + +If Salt is already installed on the system the user will be prompted to remove +the previous installation. Click ``OK`` to uninstall Salt without removing the +configuration, PKI information, or cached files. Click ``Cancel`` to abort the +installation before making any modifications to the system. + After the Welcome and the License Agreement, the installer asks for two bits of information to configure the minion; the master hostname and the minion name. -The installer will update the minion config with these options. If the installer -finds an existing minion config file, these fields will be populated with values -from the existing config. +The installer will update the minion config with these options. + +If the installer finds an existing minion config file, these fields will be +populated with values from the existing config, but they will be grayed out. +There will also be a checkbox to use existing config. If you continue, the +existing config will be used. If the checkbox is unchecked, default values are +displayed and can be changed. If you continue the existing config file in +``c:\salt\conf`` will be removed along with the ``c:\salt\conf\minion.d` +directory. The values entered will be used with the default config. The final page allows you to start the minion service and optionally change its startup type. By default, the minion is set to ``Automatic``. You can change the @@ -71,11 +87,6 @@ be managed there or from the command line like any other Windows service. sc start salt-minion net start salt-minion -.. note:: - If the minion won't start, you may need to install the Microsoft Visual C++ - 2008 x64 SP1 redistributable. Allow all Windows updates to run salt-minion - smoothly. - Installation Prerequisites -------------------------- @@ -96,15 +107,29 @@ Minion silently: ========================= ===================================================== Option Description ========================= ===================================================== -``/minion-name=`` A string value to set the minion name. Default is - 'hostname' ``/master=`` A string value to set the IP address or host name of - the master. Default value is 'salt' + the master. Default value is 'salt'. You can pass a + single master or a comma-separated list of masters. + Setting the master will replace existing config with + the default config. Cannot be used in conjunction + with ``/use-existing-config`` +``/minion-name=`` A string value to set the minion name. Default is + 'hostname'. Setting the minion name will replace + existing config with the default config. Cannot be + used in conjunction with ``/use-existing-config`` ``/start-minion=`` Either a 1 or 0. '1' will start the salt-minion service, '0' will not. Default is to start the - service after installation. + service after installation ``/start-minion-delayed`` Set the minion start type to ``Automatic (Delayed Start)`` +``/use-existing-config`` Either a 1 or 0. '1' will use the existing config if + present. '0' will replace existing config with the + default config. Default is '1'. If this is set to '1' + values passed in ``/master`` and ``/minion-name`` + will be ignored +``/S`` Runs the installation silently. Uses the above + settings or the defaults +``/?`` Displays command line help ========================= ===================================================== .. note:: From 78124181cd2b1bee2cabbe4b1de990589b56b779 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 22 Sep 2017 12:23:11 -0600 Subject: [PATCH 052/184] Fix typo --- doc/topics/installation/windows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/installation/windows.rst b/doc/topics/installation/windows.rst index 68ded4a510..080ee59a9a 100644 --- a/doc/topics/installation/windows.rst +++ b/doc/topics/installation/windows.rst @@ -48,7 +48,7 @@ Salt Minion Installation If the system is missing the appropriate version of the Visual C++ Redistributable (vcredist) the user will be prompted to install it. Click ``OK`` to install the vcredist. Click ``Cancel`` to abort the installation without -making modifications to the systeml. +making modifications to the system. If Salt is already installed on the system the user will be prompted to remove the previous installation. Click ``OK`` to uninstall Salt without removing the From e22b2a48564ee841f1d2098bf7a16d265ccc863e Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 25 Sep 2017 09:28:48 -0600 Subject: [PATCH 053/184] Fix some grammer issues in the docs --- doc/topics/installation/windows.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/topics/installation/windows.rst b/doc/topics/installation/windows.rst index 080ee59a9a..f2ec33e122 100644 --- a/doc/topics/installation/windows.rst +++ b/doc/topics/installation/windows.rst @@ -61,9 +61,9 @@ The installer will update the minion config with these options. If the installer finds an existing minion config file, these fields will be populated with values from the existing config, but they will be grayed out. -There will also be a checkbox to use existing config. If you continue, the +There will also be a checkbox to use the existing config. If you continue, the existing config will be used. If the checkbox is unchecked, default values are -displayed and can be changed. If you continue the existing config file in +displayed and can be changed. If you continue, the existing config file in ``c:\salt\conf`` will be removed along with the ``c:\salt\conf\minion.d` directory. The values entered will be used with the default config. From 601d8bc5af2764f678a0cafc5fd551ba45eb3492 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 26 Sep 2017 14:33:23 -0600 Subject: [PATCH 054/184] Add release notes --- doc/topics/releases/oxygen.rst | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/doc/topics/releases/oxygen.rst b/doc/topics/releases/oxygen.rst index ffe489b903..74e559e994 100644 --- a/doc/topics/releases/oxygen.rst +++ b/doc/topics/releases/oxygen.rst @@ -116,11 +116,36 @@ The ``state_output`` parameter now supports ``full_id``, ``changes_id`` and ``te Just like ``mixed_id``, these use the state ID as name in the highstate output. For more information on these output modes, see the docs for the :mod:`Highstate Outputter `. +Windows Installer: Changes to existing config handling +------------------------------------------------------ +Behavior with existing configuration has changed. With previous installers the +existing config was used and the master and minion id could be modified via the +installer. It was problematic in that it didn't account for configuration that +may be defined in the ``minion.d`` directory. This change gives you the option +via a checkbox to either use the existing config with out changes or the default +config using values you pass to the installer. If you choose to use the existing +config then no changes are made. If not, the existing config is deleted, to +include the ``minion.d`` directory, and the default config is used. A +command-line switch (``/use-existing-config``) has also been added to control +this behavior. + +Windows Installer: Multi-master configuration +--------------------------------------------- +The installer now has the ability to apply a multi-master configuration either +from the gui or the command line. The ``master`` field in the gui can accept +either a single master or a comma-separated list of masters. The command-line +switch (``/master=``) can accept the same. + +Windows Installer: Command-line help +------------------------------------ +The Windows installer will now display command-line help when a help switch +(``/?``) is passed. + Salt Cloud Features -------------------- +=================== Pre-Flight Commands -=================== +------------------- Support has been added for specified "preflight commands" to run on a VM before the deploy script is run. These must be defined as a list in a cloud configuration @@ -344,7 +369,7 @@ Solaris Logical Domains In Virtual Grain ---------------------------------------- Support has been added to the ``virtual`` grain for detecting Solaris LDOMs -running on T-Series SPARC hardware. The ``virtual_subtype`` grain is +running on T-Series SPARC hardware. The ``virtual_subtype`` grain is populated as a list of domain roles. Lists of comments in state returns @@ -359,7 +384,7 @@ Beacon configuration changes In order to remain consistent and to align with other Salt components such as states, support for configuring beacons using dictionary based configuration has been deprecated -in favor of list based configuration. All beacons have a validation function which will +in favor of list based configuration. All beacons have a validation function which will check the configuration for the correct format and only load if the validation passes. - ``avahi_announce`` beacon From a0972704e57be428779e0be5b40ef7baa80cb447 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 26 Sep 2017 15:11:20 -0600 Subject: [PATCH 055/184] Fix bad headings --- doc/topics/releases/oxygen.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/topics/releases/oxygen.rst b/doc/topics/releases/oxygen.rst index 74e559e994..45e00c86ae 100644 --- a/doc/topics/releases/oxygen.rst +++ b/doc/topics/releases/oxygen.rst @@ -142,10 +142,10 @@ The Windows installer will now display command-line help when a help switch (``/?``) is passed. Salt Cloud Features -=================== +------------------- Pre-Flight Commands -------------------- +=================== Support has been added for specified "preflight commands" to run on a VM before the deploy script is run. These must be defined as a list in a cloud configuration From e41e3d76beba9cc3fdad00265b07abe43af6ce97 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 16 Oct 2017 13:00:11 -0600 Subject: [PATCH 056/184] Typo fix --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index b75679cd9a..ff255b5aec 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -18,4 +18,4 @@ Yes/No Please review [Salt's Contributing Guide](https://docs.saltstack.com/en/latest/topics/development/contributing.html) for best practices. -See GitHub's [page on GPG signing](https://help.github.com/articles/signing-commits-using-gpg/) for more information signing commits with GPG. +See GitHub's [page on GPG signing](https://help.github.com/articles/signing-commits-using-gpg/) for more information about signing commits with GPG. From a685f458ed7d5f5f3edf7a8c263b3c94ef41c6da Mon Sep 17 00:00:00 2001 From: Pratik Bandarkar Date: Sat, 14 Oct 2017 00:58:58 +0000 Subject: [PATCH 057/184] add "auto_delete" option to "attach_disk" and "create_attach_volumes" for GCP. fixes #44101 - This fix will allow user to specify "auto_delete" option with attach_disk or "create_attach_volumes" function --- salt/cloud/clouds/gce.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/salt/cloud/clouds/gce.py b/salt/cloud/clouds/gce.py index 6e26cf8b95..8f4325c41b 100644 --- a/salt/cloud/clouds/gce.py +++ b/salt/cloud/clouds/gce.py @@ -2080,6 +2080,7 @@ def attach_disk(name=None, kwargs=None, call=None): disk_name = kwargs['disk_name'] mode = kwargs.get('mode', 'READ_WRITE').upper() boot = kwargs.get('boot', False) + auto_delete = kwargs.get('auto_delete', False) if boot and boot.lower() in ['true', 'yes', 'enabled']: boot = True else: @@ -2109,7 +2110,8 @@ def attach_disk(name=None, kwargs=None, call=None): transport=__opts__['transport'] ) - result = conn.attach_volume(node, disk, ex_mode=mode, ex_boot=boot) + result = conn.attach_volume(node, disk, ex_mode=mode, ex_boot=boot, + ex_auto_delete=auto_delete) __utils__['cloud.fire_event']( 'event', @@ -2389,6 +2391,8 @@ def create_attach_volumes(name, kwargs, call=None): 'type': The disk type, either pd-standard or pd-ssd. Optional, defaults to pd-standard. 'image': An image to use for this new disk. Optional. 'snapshot': A snapshot to use for this new disk. Optional. + 'auto_delete': An option(bool) to keep or remove the disk upon + instance deletion. Optional, defaults to False. Volumes are attached in the order in which they are given, thus on a new node the first volume will be /dev/sdb, the second /dev/sdc, and so on. @@ -2415,7 +2419,8 @@ def create_attach_volumes(name, kwargs, call=None): 'size': volume['size'], 'type': volume['type'], 'image': volume['image'], - 'snapshot': volume['snapshot'] + 'snapshot': volume['snapshot'], + 'auto_delete': volume.get('auto_delete', False) } create_disk(volume_dict, 'function') From 351d16840bc6881a584a205f90853422ba4ab2d8 Mon Sep 17 00:00:00 2001 From: Sergey Kizunov Date: Mon, 16 Oct 2017 15:23:36 -0500 Subject: [PATCH 058/184] Move strip_uri to salt/utils/pkg/deb.py Signed-off-by: Sergey Kizunov --- salt/modules/aptpkg.py | 19 +++---------------- salt/states/pkgrepo.py | 15 +-------------- salt/utils/pkg/deb.py | 12 ++++++++++++ 3 files changed, 16 insertions(+), 30 deletions(-) diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py index 01c0548144..f781db28c9 100644 --- a/salt/modules/aptpkg.py +++ b/salt/modules/aptpkg.py @@ -29,7 +29,6 @@ import json import yaml # pylint: disable=no-name-in-module,import-error,redefined-builtin import salt.ext.six as six -from salt.ext.six.moves import range from salt.ext.six.moves.urllib.error import HTTPError from salt.ext.six.moves.urllib.request import Request as _Request, urlopen as _urlopen # pylint: enable=no-name-in-module,import-error,redefined-builtin @@ -1558,7 +1557,7 @@ def _consolidate_repo_sources(sources): combined_comps = set(repo.comps).union(set(combined.comps)) consolidated[key].comps = list(combined_comps) else: - consolidated[key] = sourceslist.SourceEntry(_strip_uri(repo.line)) + consolidated[key] = sourceslist.SourceEntry(salt.utils.pkg.deb.strip_uri(repo.line)) if repo.file != base_file: delete_files.add(repo.file) @@ -1666,7 +1665,7 @@ def list_repos(): repo['dist'] = source.dist repo['type'] = source.type repo['uri'] = source.uri.rstrip('/') - repo['line'] = _strip_uri(source.line.strip()) + repo['line'] = salt.utils.pkg.deb.strip_uri(source.line.strip()) repo['architectures'] = getattr(source, 'architectures', []) repos.setdefault(source.uri, []).append(repo) return repos @@ -2412,18 +2411,6 @@ def file_dict(*packages): return __salt__['lowpkg.file_dict'](*packages) -def _strip_uri(repo): - ''' - Remove the trailing slash from the URI in a repo definition - ''' - splits = repo.split() - for idx in range(len(splits)): - if any(splits[idx].startswith(x) - for x in ('http://', 'https://', 'ftp://')): - splits[idx] = splits[idx].rstrip('/') - return ' '.join(splits) - - def expand_repo_def(**kwargs): ''' Take a repository definition and expand it to the full pkg repository dict @@ -2439,7 +2426,7 @@ def expand_repo_def(**kwargs): _check_apt() sanitized = {} - repo = _strip_uri(kwargs['repo']) + repo = salt.utils.pkg.deb.strip_uri(kwargs['repo']) if repo.startswith('ppa:') and __grains__['os'] in ('Ubuntu', 'Mint', 'neon'): dist = __grains__['lsb_distrib_codename'] owner_name, ppa_name = repo[4:].split('/', 1) diff --git a/salt/states/pkgrepo.py b/salt/states/pkgrepo.py index f802711a8d..93fcc6e068 100644 --- a/salt/states/pkgrepo.py +++ b/salt/states/pkgrepo.py @@ -92,7 +92,6 @@ import sys # Import salt libs from salt.exceptions import CommandExecutionError, SaltInvocationError -from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin from salt.state import STATE_INTERNAL_KEYWORDS as _STATE_INTERNAL_KEYWORDS import salt.utils import salt.utils.pkg.deb @@ -106,18 +105,6 @@ def __virtual__(): return 'pkg.mod_repo' in __salt__ -def _strip_uri(repo): - ''' - Remove the trailing slash from the URI in a repo definition - ''' - splits = repo.split() - for idx in range(len(splits)): - if any(splits[idx].startswith(x) - for x in ('http://', 'https://', 'ftp://')): - splits[idx] = splits[idx].rstrip('/') - return ' '.join(splits) - - def managed(name, ppa=None, **kwargs): ''' This state manages software package repositories. Currently, :mod:`yum @@ -381,7 +368,7 @@ def managed(name, ppa=None, **kwargs): sanitizedkwargs = kwargs if os_family == 'debian': - repo = _strip_uri(repo) + repo = salt.utils.pkg.deb.strip_uri(repo) if pre: for kwarg in sanitizedkwargs: diff --git a/salt/utils/pkg/deb.py b/salt/utils/pkg/deb.py index 7cd170a4a3..512d371bec 100644 --- a/salt/utils/pkg/deb.py +++ b/salt/utils/pkg/deb.py @@ -26,3 +26,15 @@ def combine_comments(comments): else: comments = [comments] return ' '.join(comments).strip() + + +def strip_uri(repo): + ''' + Remove the trailing slash from the URI in a repo definition + ''' + splits = repo.split() + for idx in range(len(splits)): + if any(splits[idx].startswith(x) + for x in ('http://', 'https://', 'ftp://')): + splits[idx] = splits[idx].rstrip('/') + return ' '.join(splits) From 3e962252108c86b760f8204c1d58642055a3eaca Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Sun, 15 Oct 2017 14:59:17 -0500 Subject: [PATCH 059/184] Use one salt.utils.gitfs.GitFS instance per thread This reduces some of the overhead caused by many concurrent fileclient requests from minions. Additionally, initializing remotes has been folded into the GitBase class' dunder init. Instantiating an instance and initializing its remotes were initially split so that an object could simply be created with the master opts so that the configuration was loaded and nothing else, allowing for certain cases (like clearing the cache files) where we didn't need to actually initialize the remotes. But this both A) presents a problem when the instance being used is a singleton, as you don't want to be re-initializing the remotes all the time, and B) suppressing initialization can be (and is now being) done via a new argument to the dunder init. --- salt/daemons/masterapi.py | 9 ++- salt/fileserver/gitfs.py | 68 ++++++------------- salt/master.py | 8 +-- salt/pillar/__init__.py | 8 +-- salt/pillar/git_pillar.py | 15 ++-- salt/runners/cache.py | 30 ++++---- salt/runners/git_pillar.py | 9 +-- salt/runners/winrepo.py | 11 +-- salt/utils/gitfs.py | 102 +++++++++++++++++++++++++--- tests/unit/fileserver/test_gitfs.py | 87 +++++++++++++++--------- tests/unit/utils/test_gitfs.py | 24 ++++--- 11 files changed, 227 insertions(+), 144 deletions(-) diff --git a/salt/daemons/masterapi.py b/salt/daemons/masterapi.py index 4d5a8a1c06..12efc0c935 100644 --- a/salt/daemons/masterapi.py +++ b/salt/daemons/masterapi.py @@ -69,12 +69,11 @@ def init_git_pillar(opts): for opts_dict in [x for x in opts.get('ext_pillar', [])]: if 'git' in opts_dict: try: - pillar = salt.utils.gitfs.GitPillar(opts) - pillar.init_remotes( + pillar = salt.utils.gitfs.GitPillar( + opts, opts_dict['git'], - git_pillar.PER_REMOTE_OVERRIDES, - git_pillar.PER_REMOTE_ONLY - ) + per_remote_overrides=git_pillar.PER_REMOTE_OVERRIDES, + per_remote_only=git_pillar.PER_REMOTE_ONLY) ret.append(pillar) except FileserverConfigError: if opts.get('git_pillar_verify_config', True): diff --git a/salt/fileserver/gitfs.py b/salt/fileserver/gitfs.py index 1182ec69be..477e6c40b2 100644 --- a/salt/fileserver/gitfs.py +++ b/salt/fileserver/gitfs.py @@ -71,6 +71,15 @@ log = logging.getLogger(__name__) __virtualname__ = 'git' +def _gitfs(init_remotes=True): + return salt.utils.gitfs.GitFS( + __opts__, + __opts__['gitfs_remotes'], + per_remote_overrides=PER_REMOTE_OVERRIDES, + per_remote_only=PER_REMOTE_ONLY, + init_remotes=init_remotes) + + def __virtual__(): ''' Only load if the desired provider module is present and gitfs is enabled @@ -79,7 +88,7 @@ def __virtual__(): if __virtualname__ not in __opts__['fileserver_backend']: return False try: - salt.utils.gitfs.GitFS(__opts__) + _gitfs(init_remotes=False) # Initialization of the GitFS object did not fail, so we know we have # valid configuration syntax and that a valid provider was detected. return __virtualname__ @@ -92,18 +101,14 @@ def clear_cache(): ''' Completely clear gitfs cache ''' - gitfs = salt.utils.gitfs.GitFS(__opts__) - return gitfs.clear_cache() + return _gitfs(init_remotes=False).clear_cache() def clear_lock(remote=None, lock_type='update'): ''' Clear update.lk ''' - gitfs = salt.utils.gitfs.GitFS(__opts__) - gitfs.init_remotes(__opts__['gitfs_remotes'], - PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY) - return gitfs.clear_lock(remote=remote, lock_type=lock_type) + return _gitfs().clear_lock(remote=remote, lock_type=lock_type) def lock(remote=None): @@ -114,30 +119,21 @@ def lock(remote=None): information, or a pattern. If the latter, then remotes for which the URL matches the pattern will be locked. ''' - gitfs = salt.utils.gitfs.GitFS(__opts__) - gitfs.init_remotes(__opts__['gitfs_remotes'], - PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY) - return gitfs.lock(remote=remote) + return _gitfs().lock(remote=remote) def update(): ''' Execute a git fetch on all of the repos ''' - gitfs = salt.utils.gitfs.GitFS(__opts__) - gitfs.init_remotes(__opts__['gitfs_remotes'], - PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY) - gitfs.update() + _gitfs().update() def envs(ignore_cache=False): ''' Return a list of refs that can be used as environments ''' - gitfs = salt.utils.gitfs.GitFS(__opts__) - gitfs.init_remotes(__opts__['gitfs_remotes'], - PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY) - return gitfs.envs(ignore_cache=ignore_cache) + return _gitfs().envs(ignore_cache=ignore_cache) def find_file(path, tgt_env='base', **kwargs): # pylint: disable=W0613 @@ -145,10 +141,7 @@ def find_file(path, tgt_env='base', **kwargs): # pylint: disable=W0613 Find the first file to match the path and ref, read the file out of git and send the path to the newly cached file ''' - gitfs = salt.utils.gitfs.GitFS(__opts__) - gitfs.init_remotes(__opts__['gitfs_remotes'], - PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY) - return gitfs.find_file(path, tgt_env=tgt_env, **kwargs) + return _gitfs().find_file(path, tgt_env=tgt_env, **kwargs) def init(): @@ -156,29 +149,21 @@ def init(): Initialize remotes. This is only used by the master's pre-flight checks, and is not invoked by GitFS. ''' - gitfs = salt.utils.gitfs.GitFS(__opts__) - gitfs.init_remotes(__opts__['gitfs_remotes'], - PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY) + _gitfs() def serve_file(load, fnd): ''' Return a chunk from a file based on the data received ''' - gitfs = salt.utils.gitfs.GitFS(__opts__) - gitfs.init_remotes(__opts__['gitfs_remotes'], - PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY) - return gitfs.serve_file(load, fnd) + return _gitfs().serve_file(load, fnd) def file_hash(load, fnd): ''' Return a file hash, the hash type is set in the master config file ''' - gitfs = salt.utils.gitfs.GitFS(__opts__) - gitfs.init_remotes(__opts__['gitfs_remotes'], - PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY) - return gitfs.file_hash(load, fnd) + return _gitfs().file_hash(load, fnd) def file_list(load): @@ -186,10 +171,7 @@ def file_list(load): Return a list of all files on the file server in a specified environment (specified as a key within the load dict). ''' - gitfs = salt.utils.gitfs.GitFS(__opts__) - gitfs.init_remotes(__opts__['gitfs_remotes'], - PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY) - return gitfs.file_list(load) + return _gitfs().file_list(load) def file_list_emptydirs(load): # pylint: disable=W0613 @@ -204,17 +186,11 @@ def dir_list(load): ''' Return a list of all directories on the master ''' - gitfs = salt.utils.gitfs.GitFS(__opts__) - gitfs.init_remotes(__opts__['gitfs_remotes'], - PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY) - return gitfs.dir_list(load) + return _gitfs().dir_list(load) def symlink_list(load): ''' Return a dict of all symlinks based on a given path in the repo ''' - gitfs = salt.utils.gitfs.GitFS(__opts__) - gitfs.init_remotes(__opts__['gitfs_remotes'], - PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY) - return gitfs.symlink_list(load) + return _gitfs().symlink_list(load) diff --git a/salt/master.py b/salt/master.py index 05c0cc704b..1ab22c3334 100644 --- a/salt/master.py +++ b/salt/master.py @@ -487,11 +487,11 @@ class Master(SMaster): for repo in git_pillars: new_opts[u'ext_pillar'] = [repo] try: - git_pillar = salt.utils.gitfs.GitPillar(new_opts) - git_pillar.init_remotes( + git_pillar = salt.utils.gitfs.GitPillar( + new_opts, repo[u'git'], - salt.pillar.git_pillar.PER_REMOTE_OVERRIDES, - salt.pillar.git_pillar.PER_REMOTE_ONLY) + per_remote_overrides=salt.pillar.git_pillar.PER_REMOTE_OVERRIDES, + per_remote_only=salt.pillar.git_pillar.PER_REMOTE_ONLY) except FileserverConfigError as exc: critical_errors.append(exc.strerror) finally: diff --git a/salt/pillar/__init__.py b/salt/pillar/__init__.py index 0848d76309..37cb1bd98b 100644 --- a/salt/pillar/__init__.py +++ b/salt/pillar/__init__.py @@ -891,11 +891,11 @@ class Pillar(object): # Avoid circular import import salt.utils.gitfs import salt.pillar.git_pillar - git_pillar = salt.utils.gitfs.GitPillar(self.opts) - git_pillar.init_remotes( + git_pillar = salt.utils.gitfs.GitPillar( + self.opts, self.ext['git'], - salt.pillar.git_pillar.PER_REMOTE_OVERRIDES, - salt.pillar.git_pillar.PER_REMOTE_ONLY) + per_remote_overrides=salt.pillar.git_pillar.PER_REMOTE_OVERRIDES, + per_remote_only=salt.pillar.git_pillar.PER_REMOTE_ONLY) git_pillar.fetch_remotes() except TypeError: # Handle malformed ext_pillar diff --git a/salt/pillar/git_pillar.py b/salt/pillar/git_pillar.py index fec485263d..732183a089 100644 --- a/salt/pillar/git_pillar.py +++ b/salt/pillar/git_pillar.py @@ -348,12 +348,6 @@ from salt.ext import six PER_REMOTE_OVERRIDES = ('env', 'root', 'ssl_verify', 'refspecs') PER_REMOTE_ONLY = ('name', 'mountpoint') -# Fall back to default per-remote-only. This isn't technically needed since -# salt.utils.gitfs.GitBase.init_remotes() will default to -# salt.utils.gitfs.PER_REMOTE_ONLY for this value, so this is mainly for -# runners and other modules that import salt.pillar.git_pillar. -PER_REMOTE_ONLY = salt.utils.gitfs.PER_REMOTE_ONLY - # Set up logging log = logging.getLogger(__name__) @@ -371,7 +365,7 @@ def __virtual__(): return False try: - salt.utils.gitfs.GitPillar(__opts__) + salt.utils.gitfs.GitPillar(__opts__, init_remotes=False) # Initialization of the GitPillar object did not fail, so we # know we have valid configuration syntax and that a valid # provider was detected. @@ -387,8 +381,11 @@ def ext_pillar(minion_id, pillar, *repos): # pylint: disable=unused-argument opts = copy.deepcopy(__opts__) opts['pillar_roots'] = {} opts['__git_pillar'] = True - git_pillar = salt.utils.gitfs.GitPillar(opts) - git_pillar.init_remotes(repos, PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY) + git_pillar = salt.utils.gitfs.GitPillar( + opts, + repos, + per_remote_overrides=PER_REMOTE_OVERRIDES, + per_remote_only=PER_REMOTE_ONLY) if __opts__.get('__role') == 'minion': # If masterless, fetch the remotes. We'll need to remove this once # we make the minion daemon able to run standalone. diff --git a/salt/runners/cache.py b/salt/runners/cache.py index 459ff325ea..cbd7475853 100644 --- a/salt/runners/cache.py +++ b/salt/runners/cache.py @@ -328,11 +328,14 @@ def clear_git_lock(role, remote=None, **kwargs): salt.utils.args.invalid_kwargs(kwargs) if role == 'gitfs': - git_objects = [salt.utils.gitfs.GitFS(__opts__)] - git_objects[0].init_remotes( - __opts__['gitfs_remotes'], - salt.fileserver.gitfs.PER_REMOTE_OVERRIDES, - salt.fileserver.gitfs.PER_REMOTE_ONLY) + git_objects = [ + salt.utils.gitfs.GitFS( + __opts__, + __opts__['gitfs_remotes'], + per_remote_overrides=salt.fileserver.gitfs.PER_REMOTE_OVERRIDES, + per_remote_only=salt.fileserver.gitfs.PER_REMOTE_ONLY + ) + ] elif role == 'git_pillar': git_objects = [] for ext_pillar in __opts__['ext_pillar']: @@ -340,11 +343,11 @@ def clear_git_lock(role, remote=None, **kwargs): if key == 'git': if not isinstance(ext_pillar['git'], list): continue - obj = salt.utils.gitfs.GitPillar(__opts__) - obj.init_remotes( + obj = salt.utils.gitfs.GitPillar( + __opts__, ext_pillar['git'], - salt.pillar.git_pillar.PER_REMOTE_OVERRIDES, - salt.pillar.git_pillar.PER_REMOTE_ONLY) + per_remote_overrides=salt.pillar.git_pillar.PER_REMOTE_OVERRIDES, + per_remote_only=salt.pillar.git_pillar.PER_REMOTE_ONLY) git_objects.append(obj) elif role == 'winrepo': winrepo_dir = __opts__['winrepo_dir'] @@ -355,11 +358,12 @@ def clear_git_lock(role, remote=None, **kwargs): (winrepo_remotes, winrepo_dir), (__opts__['winrepo_remotes_ng'], __opts__['winrepo_dir_ng']) ): - obj = salt.utils.gitfs.WinRepo(__opts__, base_dir) - obj.init_remotes( + obj = salt.utils.gitfs.WinRepo( + __opts__, remotes, - salt.runners.winrepo.PER_REMOTE_OVERRIDES, - salt.runners.winrepo.PER_REMOTE_ONLY) + per_remote_overrides=salt.runners.winrepo.PER_REMOTE_OVERRIDES, + per_remote_only=salt.runners.winrepo.PER_REMOTE_ONLY, + cache_root=base_dir) git_objects.append(obj) else: raise SaltInvocationError('Invalid role \'{0}\''.format(role)) diff --git a/salt/runners/git_pillar.py b/salt/runners/git_pillar.py index 6826268076..984c7da8cc 100644 --- a/salt/runners/git_pillar.py +++ b/salt/runners/git_pillar.py @@ -66,10 +66,11 @@ def update(branch=None, repo=None): if pillar_type != 'git': continue pillar_conf = ext_pillar[pillar_type] - pillar = salt.utils.gitfs.GitPillar(__opts__) - pillar.init_remotes(pillar_conf, - salt.pillar.git_pillar.PER_REMOTE_OVERRIDES, - salt.pillar.git_pillar.PER_REMOTE_ONLY) + pillar = salt.utils.gitfs.GitPillar( + __opts__, + pillar_conf, + per_remote_overrides=salt.pillar.git_pillar.PER_REMOTE_OVERRIDES, + per_remote_only=salt.pillar.git_pillar.PER_REMOTE_ONLY) for remote in pillar.remotes: # Skip this remote if it doesn't match the search criteria if branch is not None: diff --git a/salt/runners/winrepo.py b/salt/runners/winrepo.py index 1e73974c4e..4aa20d2b35 100644 --- a/salt/runners/winrepo.py +++ b/salt/runners/winrepo.py @@ -32,7 +32,7 @@ log = logging.getLogger(__name__) PER_REMOTE_OVERRIDES = ('ssl_verify', 'refspecs') # Fall back to default per-remote-only. This isn't technically needed since -# salt.utils.gitfs.GitBase.init_remotes() will default to +# salt.utils.gitfs.GitBase.__init__ will default to # salt.utils.gitfs.PER_REMOTE_ONLY for this value, so this is mainly for # runners and other modules that import salt.runners.winrepo. PER_REMOTE_ONLY = salt.utils.gitfs.PER_REMOTE_ONLY @@ -216,9 +216,12 @@ def update_git_repos(opts=None, clean=False, masterless=False): else: # New winrepo code utilizing salt.utils.gitfs try: - winrepo = salt.utils.gitfs.WinRepo(opts, base_dir) - winrepo.init_remotes( - remotes, PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY) + winrepo = salt.utils.gitfs.WinRepo( + opts, + remotes, + per_remote_overrides=PER_REMOTE_OVERRIDES, + per_remote_only=PER_REMOTE_ONLY, + cache_root=base_dir) winrepo.fetch_remotes() # Since we're not running update(), we need to manually call # clear_old_remotes() to remove directories from remotes that diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py index ae028463a0..2f4b119837 100644 --- a/salt/utils/gitfs.py +++ b/salt/utils/gitfs.py @@ -17,6 +17,8 @@ import shutil import stat import subprocess import time +import tornado.ioloop +import weakref from datetime import datetime # Import salt libs @@ -1923,12 +1925,47 @@ class GitBase(object): ''' Base class for gitfs/git_pillar ''' - def __init__(self, opts, git_providers=None, cache_root=None): + def __init__(self, opts, remotes=None, per_remote_overrides=(), + per_remote_only=PER_REMOTE_ONLY, git_providers=None, + cache_root=None, init_remotes=True): ''' IMPORTANT: If specifying a cache_root, understand that this is also where the remotes will be cloned. A non-default cache_root is only really designed right now for winrepo, as its repos need to be checked out into the winrepo locations and not within the cachedir. + + As of the Oxygen release cycle, the classes used to interface with + Pygit2 and GitPython can be overridden by passing the git_providers + argument when spawning a class instance. This allows for one to write + classes which inherit from salt.utils.gitfs.Pygit2 or + salt.utils.gitfs.GitPython, and then direct one of the GitBase + subclasses (GitFS, GitPillar, WinRepo) to use the custom class. For + example: + + .. code-block:: Python + + import salt.utils.gitfs + from salt.fileserver.gitfs import PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY + + class CustomPygit2(salt.utils.gitfs.Pygit2): + def fetch_remotes(self): + ... + Alternate fetch behavior here + ... + + git_providers = { + 'pygit2': CustomPygit2, + 'gitpython': salt.utils.gitfs.GitPython, + } + + gitfs = salt.utils.gitfs.GitFS( + __opts__, + __opts__['gitfs_remotes'], + per_remote_overrides=PER_REMOTE_OVERRIDES, + per_remote_only=PER_REMOTE_ONLY, + git_providers=git_providers) + + gitfs.fetch_remotes() ''' self.opts = opts self.git_providers = git_providers if git_providers is not None \ @@ -1944,8 +1981,13 @@ class GitBase(object): self.hash_cachedir = salt.utils.path.join(self.cache_root, 'hash') self.file_list_cachedir = salt.utils.path.join( self.opts['cachedir'], 'file_lists', self.role) + if init_remotes: + self.init_remotes( + remotes if remotes is not None else [], + per_remote_overrides, + per_remote_only) - def init_remotes(self, remotes, per_remote_overrides, + def init_remotes(self, remotes, per_remote_overrides=(), per_remote_only=PER_REMOTE_ONLY): ''' Initialize remotes @@ -2469,9 +2511,51 @@ class GitFS(GitBase): ''' Functionality specific to the git fileserver backend ''' - def __init__(self, opts): - self.role = 'gitfs' - super(GitFS, self).__init__(opts) + role = 'gitfs' + instance_map = weakref.WeakKeyDictionary() + + def __new__(cls, opts, remotes=None, per_remote_overrides=(), + per_remote_only=PER_REMOTE_ONLY, git_providers=None, + cache_root=None, init_remotes=True): + ''' + If we are not initializing remotes (such as in cases where we just want + to load the config so that we can run clear_cache), then just return a + new __init__'ed object. Otherwise, check the instance map and re-use an + instance if one exists for the current process. Weak references are + used to ensure that we garbage collect instances for threads which have + exited. + ''' + # No need to get the ioloop reference if we're not initializing remotes + io_loop = tornado.ioloop.IOLoop.current() if init_remotes else None + if not init_remotes or io_loop not in cls.instance_map: + # We only evaluate the second condition in this if statement if + # we're initializing remotes, so we won't get here unless io_loop + # is something other than None. + obj = object.__new__(cls) + super(GitFS, obj).__init__( + opts, + remotes if remotes is not None else [], + per_remote_overrides=per_remote_overrides, + per_remote_only=per_remote_only, + git_providers=git_providers if git_providers is not None + else GIT_PROVIDERS, + cache_root=cache_root, + init_remotes=init_remotes) + if not init_remotes: + log.debug('Created gitfs object with uninitialized remotes') + else: + log.debug('Created gitfs object for process %s', os.getpid()) + # Add to the instance map so we can re-use later + cls.instance_map[io_loop] = obj + return obj + log.debug('Re-using gitfs object for process %s', os.getpid()) + return cls.instance_map[io_loop] + + def __init__(self, opts, remotes, per_remote_overrides=(), # pylint: disable=super-init-not-called + per_remote_only=PER_REMOTE_ONLY, git_providers=None, + cache_root=None, init_remotes=True): + # Initialization happens above in __new__(), so don't do anything here + pass def dir_list(self, load): ''' @@ -2753,9 +2837,7 @@ class GitPillar(GitBase): ''' Functionality specific to the git external pillar ''' - def __init__(self, opts): - self.role = 'git_pillar' - super(GitPillar, self).__init__(opts) + role = 'git_pillar' def checkout(self): ''' @@ -2843,9 +2925,7 @@ class WinRepo(GitBase): ''' Functionality specific to the winrepo runner ''' - def __init__(self, opts, winrepo_dir): - self.role = 'winrepo' - super(WinRepo, self).__init__(opts, cache_root=winrepo_dir) + role = 'winrepo' def checkout(self): ''' diff --git a/tests/unit/fileserver/test_gitfs.py b/tests/unit/fileserver/test_gitfs.py index 64d8ca5284..bda0182eec 100644 --- a/tests/unit/fileserver/test_gitfs.py +++ b/tests/unit/fileserver/test_gitfs.py @@ -5,10 +5,12 @@ # Import Python libs from __future__ import absolute_import +import errno import os import shutil import tempfile import textwrap +import tornado.ioloop import logging import stat try: @@ -40,18 +42,26 @@ import salt.utils.win_functions log = logging.getLogger(__name__) +TMP_SOCK_DIR = tempfile.mkdtemp(dir=TMP) +TMP_REPO_DIR = os.path.join(TMP, 'gitfs_root') +INTEGRATION_BASE_FILES = os.path.join(FILES, 'file', 'base') + + +def _rmtree_error(func, path, excinfo): + os.chmod(path, stat.S_IWRITE) + func(path) + @skipIf(not HAS_GITPYTHON, 'GitPython is not installed') class GitfsConfigTestCase(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): self.tmp_cachedir = tempfile.mkdtemp(dir=TMP) - self.tmp_sock_dir = tempfile.mkdtemp(dir=TMP) return { gitfs: { '__opts__': { 'cachedir': self.tmp_cachedir, - 'sock_dir': self.tmp_sock_dir, + 'sock_dir': TMP_SOCK_DIR, 'gitfs_root': 'salt', 'fileserver_backend': ['git'], 'gitfs_base': 'master', @@ -81,9 +91,17 @@ class GitfsConfigTestCase(TestCase, LoaderModuleMockMixin): } } + @classmethod + def setUpClass(cls): + # Clear the instance map so that we make sure to create a new instance + # for this test class. + try: + del salt.utils.gitfs.GitFS.instance_map[tornado.ioloop.IOLoop.current()] + except KeyError: + pass + def tearDown(self): shutil.rmtree(self.tmp_cachedir) - shutil.rmtree(self.tmp_sock_dir) def test_per_saltenv_config(self): opts_override = textwrap.dedent(''' @@ -109,10 +127,11 @@ class GitfsConfigTestCase(TestCase, LoaderModuleMockMixin): - mountpoint: abc ''') with patch.dict(gitfs.__opts__, yaml.safe_load(opts_override)): - git_fs = salt.utils.gitfs.GitFS(gitfs.__opts__) - git_fs.init_remotes( + git_fs = salt.utils.gitfs.GitFS( + gitfs.__opts__, gitfs.__opts__['gitfs_remotes'], - gitfs.PER_REMOTE_OVERRIDES, gitfs.PER_REMOTE_ONLY) + per_remote_overrides=gitfs.PER_REMOTE_OVERRIDES, + per_remote_only=gitfs.PER_REMOTE_ONLY) # repo1 (branch: foo) # The mountpoint should take the default (from gitfs_mountpoint), while @@ -169,14 +188,12 @@ class GitFSTest(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): self.tmp_cachedir = tempfile.mkdtemp(dir=TMP) - self.tmp_sock_dir = tempfile.mkdtemp(dir=TMP) - self.tmp_repo_dir = os.path.join(TMP, 'gitfs_root') return { gitfs: { '__opts__': { 'cachedir': self.tmp_cachedir, - 'sock_dir': self.tmp_sock_dir, - 'gitfs_remotes': ['file://' + self.tmp_repo_dir], + 'sock_dir': TMP_SOCK_DIR, + 'gitfs_remotes': ['file://' + TMP_REPO_DIR], 'gitfs_root': '', 'fileserver_backend': ['git'], 'gitfs_base': 'master', @@ -206,26 +223,26 @@ class GitFSTest(TestCase, LoaderModuleMockMixin): } } - def setUp(self): - ''' - We don't want to check in another .git dir into GH because that just gets messy. - Instead, we'll create a temporary repo on the fly for the tests to examine. - ''' - if not gitfs.__virtual__(): - self.skipTest("GitFS could not be loaded. Skipping GitFS tests!") - self.integration_base_files = os.path.join(FILES, 'file', 'base') + @classmethod + def setUpClass(cls): + # Clear the instance map so that we make sure to create a new instance + # for this test class. + try: + del salt.utils.gitfs.GitFS.instance_map[tornado.ioloop.IOLoop.current()] + except KeyError: + pass # Create the dir if it doesn't already exist try: - shutil.copytree(self.integration_base_files, self.tmp_repo_dir + '/') + shutil.copytree(INTEGRATION_BASE_FILES, TMP_REPO_DIR + '/') except OSError: # We probably caught an error because files already exist. Ignore pass try: - repo = git.Repo(self.tmp_repo_dir) + repo = git.Repo(TMP_REPO_DIR) except git.exc.InvalidGitRepositoryError: - repo = git.Repo.init(self.tmp_repo_dir) + repo = git.Repo.init(TMP_REPO_DIR) if 'USERNAME' not in os.environ: try: @@ -238,9 +255,19 @@ class GitFSTest(TestCase, LoaderModuleMockMixin): '\'root\'.') os.environ['USERNAME'] = 'root' - repo.index.add([x for x in os.listdir(self.tmp_repo_dir) + repo.index.add([x for x in os.listdir(TMP_REPO_DIR) if x != '.git']) repo.index.commit('Test') + + def setUp(self): + ''' + We don't want to check in another .git dir into GH because that just + gets messy. Instead, we'll create a temporary repo on the fly for the + tests to examine. + ''' + if not gitfs.__virtual__(): + self.skipTest("GitFS could not be loaded. Skipping GitFS tests!") + self.tmp_cachedir = tempfile.mkdtemp(dir=TMP) gitfs.update() def tearDown(self): @@ -248,17 +275,11 @@ class GitFSTest(TestCase, LoaderModuleMockMixin): Remove the temporary git repository and gitfs cache directory to ensure a clean environment for each test. ''' - shutil.rmtree(self.tmp_repo_dir, onerror=self._rmtree_error) - shutil.rmtree(self.tmp_cachedir, onerror=self._rmtree_error) - shutil.rmtree(self.tmp_sock_dir, onerror=self._rmtree_error) - del self.tmp_repo_dir - del self.tmp_cachedir - del self.tmp_sock_dir - del self.integration_base_files - - def _rmtree_error(self, func, path, excinfo): - os.chmod(path, stat.S_IWRITE) - func(path) + try: + shutil.rmtree(self.tmp_cachedir, onerror=_rmtree_error) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise def test_file_list(self): ret = gitfs.file_list(LOAD) diff --git a/tests/unit/utils/test_gitfs.py b/tests/unit/utils/test_gitfs.py index 070a46fe75..89a8b4fd59 100644 --- a/tests/unit/utils/test_gitfs.py +++ b/tests/unit/utils/test_gitfs.py @@ -37,18 +37,19 @@ class TestGitFSProvider(TestCase): MagicMock(return_value=True)): with patch.object(role_class, 'verify_pygit2', MagicMock(return_value=False)): - args = [OPTS] + args = [OPTS, {}] + kwargs = {'init_remotes': False} if role_name == 'winrepo': - args.append('/tmp/winrepo-dir') + kwargs['cache_root'] = '/tmp/winrepo-dir' with patch.dict(OPTS, {key: provider}): # Try to create an instance with uppercase letters in # provider name. If it fails then a # FileserverConfigError will be raised, so no assert is # necessary. - role_class(*args) - # Now try to instantiate an instance with all lowercase - # letters. Again, no need for an assert here. - role_class(*args) + role_class(*args, **kwargs) + # Now try to instantiate an instance with all lowercase + # letters. Again, no need for an assert here. + role_class(*args, **kwargs) def test_valid_provider(self): ''' @@ -73,12 +74,13 @@ class TestGitFSProvider(TestCase): verify = 'verify_pygit2' mock2 = _get_mock(verify, provider) with patch.object(role_class, verify, mock2): - args = [OPTS] + args = [OPTS, {}] + kwargs = {'init_remotes': False} if role_name == 'winrepo': - args.append('/tmp/winrepo-dir') + kwargs['cache_root'] = '/tmp/winrepo-dir' with patch.dict(OPTS, {key: provider}): - role_class(*args) + role_class(*args, **kwargs) with patch.dict(OPTS, {key: 'foo'}): # Set the provider name to a known invalid provider @@ -86,5 +88,5 @@ class TestGitFSProvider(TestCase): self.assertRaises( FileserverConfigError, role_class, - *args - ) + *args, + **kwargs) From b4ba7ae2fccde772e60006a06c81762d396bd80d Mon Sep 17 00:00:00 2001 From: Matthew Summers Date: Mon, 9 Oct 2017 20:38:52 -0500 Subject: [PATCH 060/184] addresses issue #43307, disk.format_ to disk.format This change fixes breakage. It appears the disk.format_ func is aliased to disk.format in modules/disk.py --- salt/states/blockdev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/blockdev.py b/salt/states/blockdev.py index 694a168d4a..1b08e3641a 100644 --- a/salt/states/blockdev.py +++ b/salt/states/blockdev.py @@ -159,7 +159,7 @@ def formatted(name, fs_type='ext4', force=False, **kwargs): ret['result'] = None return ret - __salt__['disk.format_'](name, fs_type, force=force, **kwargs) + __salt__['disk.format'](name, fs_type, force=force, **kwargs) current_fs = __salt__['disk.fstype'](name) # Repeat lsblk check up to 10 times with 3s sleeping between each From bebf301976cfc7d4ddf429e790c105fb5330d52d Mon Sep 17 00:00:00 2001 From: Matthew Summers Date: Mon, 16 Oct 2017 09:47:40 -0500 Subject: [PATCH 061/184] fixed test addressing issue #43307, disk.format_ to disk.format --- tests/unit/states/blockdev_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/states/blockdev_test.py b/tests/unit/states/blockdev_test.py index defbc856cd..2bc8dc376f 100644 --- a/tests/unit/states/blockdev_test.py +++ b/tests/unit/states/blockdev_test.py @@ -80,7 +80,7 @@ class BlockdevTestCase(TestCase): mock_t = MagicMock(return_value=True) mock_e = MagicMock(return_value='') with patch.dict(blockdev.__salt__, {'cmd.run': mock_ext4, - 'disk.format_': mock_t, + 'disk.format': mock_t, 'disk.fstype': mock_e}): comt = ('{0} already formatted with '.format(name)) ret.update({'comment': comt, 'result': True}) From a89c26c88e7dd0bef4b7ca22178e5a954ecdce8a Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 12 Oct 2017 10:22:17 -0700 Subject: [PATCH 062/184] Various fixes to beacons to handle situations when beacons are stored in pillar. Updating execution module to report that beacons stored in pillar can not be managed via module. Updating add function to ensure beacon is available before attempting to add it. Updaitng unit tests to include new beacon is available check. --- salt/beacons/__init__.py | 86 ++++++++++++++++++++++-------- salt/minion.py | 3 +- salt/modules/beacons.py | 40 ++++++++++---- tests/unit/modules/test_beacons.py | 3 ++ 4 files changed, 99 insertions(+), 33 deletions(-) diff --git a/salt/beacons/__init__.py b/salt/beacons/__init__.py index 0830e39b5d..8ebe768888 100644 --- a/salt/beacons/__init__.py +++ b/salt/beacons/__init__.py @@ -199,13 +199,16 @@ class Beacon(object): else: self.opts['beacons'][name].append({'enabled': enabled_value}) - def list_beacons(self): + def list_beacons(self, include_pillar): ''' List the beacon items ''' # Fire the complete event back along with the list of beacons evt = salt.utils.event.get_event('minion', opts=self.opts) - evt.fire_event({'complete': True, 'beacons': self.opts['beacons']}, + _beacons = self.opts['beacons'] + if include_pillar and 'beacons' in self.opts['pillar']: + _beacons.update(self.opts['pillar']['beacons']) + evt.fire_event({'complete': True, 'beacons': _beacons}, tag='/salt/minion/minion_beacons_list_complete') return True @@ -236,8 +239,8 @@ class Beacon(object): del beacon_data['enabled'] valid, vcomment = self.beacons[validate_str](beacon_data) else: - log.info('Beacon %s does not have a validate' - ' function, skipping validation.', name) + vcomment = ('Beacon %s does not have a validate' + ' function, skipping validation.', name) valid = True # Fire the complete event back along with the list of beacons @@ -257,16 +260,23 @@ class Beacon(object): data = {} data[name] = beacon_data - if name in self.opts['beacons']: - log.info('Updating settings for beacon ' - 'item: %s', name) + if name in self.opts.get('pillar', {}).get('beacons', {}): + comment = ('Cannot update beacon item %s, ' + 'it is in pillar.', name) + complete = False else: - log.info('Added new beacon item %s', name) - self.opts['beacons'].update(data) + if name in self.opts['beacons']: + comment = ('Updating settings for beacon ' + 'item: %s', name) + else: + comment = ('Added new beacon item %s', name) + complete = True + self.opts['beacons'].update(data) # Fire the complete event back along with updated list of beacons evt = salt.utils.event.get_event('minion', opts=self.opts) - evt.fire_event({'complete': True, 'beacons': self.opts['beacons']}, + evt.fire_event({'complete': complete, 'comment': comment, + 'beacons': self.opts['beacons']}, tag='/salt/minion/minion_beacon_add_complete') return True @@ -279,13 +289,20 @@ class Beacon(object): data = {} data[name] = beacon_data - log.info('Updating settings for beacon ' - 'item: %s', name) - self.opts['beacons'].update(data) + if name in self.opts.get('pillar', {}).get('beacons', {}): + comment = ('Cannot modify beacon item %s, ' + 'it is in pillar.', name) + complete = False + else: + comment = ('Updating settings for beacon ' + 'item: %s', name) + complete = True + self.opts['beacons'].update(data) # Fire the complete event back along with updated list of beacons evt = salt.utils.event.get_event('minion', opts=self.opts) - evt.fire_event({'complete': True, 'beacons': self.opts['beacons']}, + evt.fire_event({'complete': complete, 'comment': comment, + 'beacons': self.opts['beacons']}, tag='/salt/minion/minion_beacon_modify_complete') return True @@ -295,13 +312,22 @@ class Beacon(object): Delete a beacon item ''' - if name in self.opts['beacons']: - log.info('Deleting beacon item %s', name) - del self.opts['beacons'][name] + if name in self.opts.get('pillar', {}).get('beacons', {}): + comment = ('Cannot delete beacon item 5s, ' + 'it is in pillar.', name) + complete = False + else: + if name in self.opts['beacons']: + del self.opts['beacons'][name] + comment = ('Deleting beacon item %s', name) + else: + comment = ('Beacon item %s not found.', name) + complete = True # Fire the complete event back along with updated list of beacons evt = salt.utils.event.get_event('minion', opts=self.opts) - evt.fire_event({'complete': True, 'beacons': self.opts['beacons']}, + evt.fire_event({'complete': complete, 'comment': comment, + 'beacons': self.opts['beacons']}, tag='/salt/minion/minion_beacon_delete_complete') return True @@ -339,11 +365,19 @@ class Beacon(object): Enable a beacon ''' - self._update_enabled(name, True) + if name in self.opts.get('pillar', {}).get('beacons', {}): + comment = ('Cannot enable beacon item %s, ' + 'it is in pillar.', name) + complete = False + else: + self._update_enabled(name, True) + comment = ('Enabling beacon item %s', name) + complete = True # Fire the complete event back along with updated list of beacons evt = salt.utils.event.get_event('minion', opts=self.opts) - evt.fire_event({'complete': True, 'beacons': self.opts['beacons']}, + evt.fire_event({'complete': complete, 'comment': comment, + 'beacons': self.opts['beacons']}, tag='/salt/minion/minion_beacon_enabled_complete') return True @@ -353,11 +387,19 @@ class Beacon(object): Disable a beacon ''' - self._update_enabled(name, False) + if name in self.opts.get('pillar', {}).get('beacons', {}): + comment = ('Cannot disable beacon item %s, ' + 'it is in pillar.', name) + complete = False + else: + self._update_enabled(name, False) + comment = ('Disabling beacon item %s', name) + complete = True # Fire the complete event back along with updated list of beacons evt = salt.utils.event.get_event('minion', opts=self.opts) - evt.fire_event({'complete': True, 'beacons': self.opts['beacons']}, + evt.fire_event({'complete': complete, 'comment': comment, + 'beacons': self.opts['beacons']}, tag='/salt/minion/minion_beacon_disabled_complete') return True diff --git a/salt/minion.py b/salt/minion.py index 9ea6c170db..d51064dc87 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -2063,6 +2063,7 @@ class Minion(MinionBase): func = data.get(u'func', None) name = data.get(u'name', None) beacon_data = data.get(u'beacon_data', None) + include_pillar = data.get(u'include_pillar', None) if func == u'add': self.beacons.add_beacon(name, beacon_data) @@ -2079,7 +2080,7 @@ class Minion(MinionBase): elif func == u'disable_beacon': self.beacons.disable_beacon(name) elif func == u'list': - self.beacons.list_beacons() + self.beacons.list_beacons(include_pillar) elif func == u'list_available': self.beacons.list_available_beacons() elif func == u'validate_beacon': diff --git a/salt/modules/beacons.py b/salt/modules/beacons.py index 7e095ed656..f5dbcbe99f 100644 --- a/salt/modules/beacons.py +++ b/salt/modules/beacons.py @@ -28,7 +28,7 @@ __func_alias__ = { } -def list_(return_yaml=True): +def list_(return_yaml=True, include_pillar=True): ''' List the beacons currently configured on the minion @@ -46,7 +46,9 @@ def list_(return_yaml=True): try: eventer = salt.utils.event.get_event('minion', opts=__opts__) - res = __salt__['event.fire']({'func': 'list'}, 'manage_beacons') + res = __salt__['event.fire']({'func': 'list', + 'include_pillar': include_pillar}, + 'manage_beacons') if res: event_ret = eventer.get_event(tag='/salt/minion/minion_beacons_list_complete', wait=30) log.debug('event_ret {0}'.format(event_ret)) @@ -133,6 +135,10 @@ def add(name, beacon_data, **kwargs): ret['comment'] = 'Beacon {0} is already configured.'.format(name) return ret + if name not in list_available(return_yaml=False): + ret['comment'] = 'Beacon {0} is not available.'.format(name) + return ret + if 'test' in kwargs and kwargs['test']: ret['result'] = True ret['comment'] = 'Beacon: {0} would be added.'.format(name) @@ -170,7 +176,10 @@ def add(name, beacon_data, **kwargs): if name in beacons and beacons[name] == beacon_data: ret['result'] = True ret['comment'] = 'Added beacon: {0}.'.format(name) - return ret + else: + ret['result'] = False + ret['comment'] = event_ret['comment'] + return ret except KeyError: # Effectively a no-op, since we can't really return without an event system ret['comment'] = 'Event module not available. Beacon add failed.' @@ -262,7 +271,10 @@ def modify(name, beacon_data, **kwargs): if name in beacons and beacons[name] == beacon_data: ret['result'] = True ret['comment'] = 'Modified beacon: {0}.'.format(name) - return ret + else: + ret['result'] = False + ret['comment'] = event_ret['comment'] + return ret except KeyError: # Effectively a no-op, since we can't really return without an event system ret['comment'] = 'Event module not available. Beacon add failed.' @@ -299,12 +311,14 @@ def delete(name, **kwargs): if res: event_ret = eventer.get_event(tag='/salt/minion/minion_beacon_delete_complete', wait=30) if event_ret and event_ret['complete']: - log.debug('== event_ret {} =='.format(event_ret)) beacons = event_ret['beacons'] if name not in beacons: ret['result'] = True ret['comment'] = 'Deleted beacon: {0}.'.format(name) return ret + else: + ret['result'] = False + ret['comment'] = event_ret['comment'] except KeyError: # Effectively a no-op, since we can't really return without an event system ret['comment'] = 'Event module not available. Beacon add failed.' @@ -327,7 +341,7 @@ def save(): ret = {'comment': [], 'result': True} - beacons = list_(return_yaml=False) + beacons = list_(return_yaml=False, include_pillar=False) # move this file into an configurable opt sfn = '{0}/{1}/beacons.conf'.format(__opts__['config_dir'], @@ -380,7 +394,7 @@ def enable(**kwargs): else: ret['result'] = False ret['comment'] = 'Failed to enable beacons on minion.' - return ret + return ret except KeyError: # Effectively a no-op, since we can't really return without an event system ret['comment'] = 'Event module not available. Beacons enable job failed.' @@ -420,7 +434,7 @@ def disable(**kwargs): else: ret['result'] = False ret['comment'] = 'Failed to disable beacons on minion.' - return ret + return ret except KeyError: # Effectively a no-op, since we can't really return without an event system ret['comment'] = 'Event module not available. Beacons enable job failed.' @@ -483,7 +497,10 @@ def enable_beacon(name, **kwargs): else: ret['result'] = False ret['comment'] = 'Failed to enable beacon {0} on minion.'.format(name) - return ret + else: + ret['result'] = False + ret['comment'] = event_ret['comment'] + return ret except KeyError: # Effectively a no-op, since we can't really return without an event system ret['comment'] = 'Event module not available. Beacon enable job failed.' @@ -536,7 +553,10 @@ def disable_beacon(name, **kwargs): else: ret['result'] = False ret['comment'] = 'Failed to disable beacon on minion.' - return ret + else: + ret['result'] = False + ret['comment'] = event_ret['comment'] + return ret except KeyError: # Effectively a no-op, since we can't really return without an event system ret['comment'] = 'Event module not available. Beacon disable job failed.' diff --git a/tests/unit/modules/test_beacons.py b/tests/unit/modules/test_beacons.py index 6706866bb9..73f012a75d 100644 --- a/tests/unit/modules/test_beacons.py +++ b/tests/unit/modules/test_beacons.py @@ -59,6 +59,9 @@ class BeaconsTestCase(TestCase, LoaderModuleMockMixin): event_returns = [{'complete': True, 'tag': '/salt/minion/minion_beacons_list_complete', 'beacons': {}}, + {'complete': True, + 'tag': '/salt/minion/minion_beacons_list_available_complete', + 'beacons': ['ps']}, {'complete': True, 'valid': True, 'vcomment': '', From 2738a124fe3eebc40c95c3560bf075ddc5b3f5b9 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 12 Oct 2017 17:49:40 -0700 Subject: [PATCH 063/184] Some cleanup of duplicated code. Fixing event comments. --- salt/beacons/__init__.py | 90 ++++++++++++++++++++++++++-------------- salt/minion.py | 4 +- salt/modules/beacons.py | 8 ++-- 3 files changed, 66 insertions(+), 36 deletions(-) diff --git a/salt/beacons/__init__.py b/salt/beacons/__init__.py index 8ebe768888..53e085f25f 100644 --- a/salt/beacons/__init__.py +++ b/salt/beacons/__init__.py @@ -199,16 +199,44 @@ class Beacon(object): else: self.opts['beacons'][name].append({'enabled': enabled_value}) - def list_beacons(self, include_pillar): + def _get_beacons(self, + include_opts=True, + include_pillar=True): + ''' + Return the beacons data structure + ''' + beacons = {} + if include_pillar: + pillar_beacons = self.opts.get('pillar', {}).get('beacons', {}) + if not isinstance(pillar_beacons, dict): + raise ValueError('Beacons must be of type dict.') + beacons.update(pillar_beacons) + if include_opts: + opts_beacons = self.opts.get('beacons', {}) + if not isinstance(opts_beacons, dict): + raise ValueError('Beacons must be of type dict.') + beacons.update(opts_beacons) + return beacons + + def list_beacons(self, where=None): ''' List the beacon items + + where: Whether to include beacon data from, either opts + or pillar, default is None which would include data + from both. + ''' + if where == 'pillar': + beacons = self._get_beacons(include_opts=False) + elif where == 'opts': + beacons = self._get_beacons(include_pillar=False) + else: + beacons = self._get_beacons() + # Fire the complete event back along with the list of beacons evt = salt.utils.event.get_event('minion', opts=self.opts) - _beacons = self.opts['beacons'] - if include_pillar and 'beacons' in self.opts['pillar']: - _beacons.update(self.opts['pillar']['beacons']) - evt.fire_event({'complete': True, 'beacons': _beacons}, + evt.fire_event({'complete': True, 'beacons': beacons}, tag='/salt/minion/minion_beacons_list_complete') return True @@ -239,8 +267,8 @@ class Beacon(object): del beacon_data['enabled'] valid, vcomment = self.beacons[validate_str](beacon_data) else: - vcomment = ('Beacon %s does not have a validate' - ' function, skipping validation.', name) + vcomment = 'Beacon {0} does not have a validate' \ + ' function, skipping validation.'.format(name) valid = True # Fire the complete event back along with the list of beacons @@ -260,16 +288,16 @@ class Beacon(object): data = {} data[name] = beacon_data - if name in self.opts.get('pillar', {}).get('beacons', {}): - comment = ('Cannot update beacon item %s, ' - 'it is in pillar.', name) + if name in self._get_beacons(include_opts=False): + comment = 'Cannot update beacon item {0}, ' \ + 'it is in pillar.'.format(name) complete = False else: if name in self.opts['beacons']: - comment = ('Updating settings for beacon ' - 'item: %s', name) + comment = 'Updating settings for beacon ' \ + 'item: {0}'.format(name) else: - comment = ('Added new beacon item %s', name) + comment = 'Added new beacon item {0}'.format(name) complete = True self.opts['beacons'].update(data) @@ -289,13 +317,13 @@ class Beacon(object): data = {} data[name] = beacon_data - if name in self.opts.get('pillar', {}).get('beacons', {}): - comment = ('Cannot modify beacon item %s, ' - 'it is in pillar.', name) + if name in self._get_beacons(include_opts=False): + comment = 'Cannot modify beacon item {0}, ' \ + 'it is in pillar.'.format(name) complete = False else: - comment = ('Updating settings for beacon ' - 'item: %s', name) + comment = 'Updating settings for beacon ' \ + 'item: {0}'.format(name) complete = True self.opts['beacons'].update(data) @@ -312,16 +340,16 @@ class Beacon(object): Delete a beacon item ''' - if name in self.opts.get('pillar', {}).get('beacons', {}): - comment = ('Cannot delete beacon item 5s, ' - 'it is in pillar.', name) + if name in self._get_beacons(include_opts=False): + comment = 'Cannot delete beacon item {0}, ' \ + 'it is in pillar.'.format(name) complete = False else: if name in self.opts['beacons']: del self.opts['beacons'][name] - comment = ('Deleting beacon item %s', name) + comment = 'Deleting beacon item {0}'.format(name) else: - comment = ('Beacon item %s not found.', name) + comment = 'Beacon item {0} not found.'.format(name) complete = True # Fire the complete event back along with updated list of beacons @@ -365,13 +393,13 @@ class Beacon(object): Enable a beacon ''' - if name in self.opts.get('pillar', {}).get('beacons', {}): - comment = ('Cannot enable beacon item %s, ' - 'it is in pillar.', name) + if name in self._get_beacons(include_opts=False): + comment = 'Cannot enable beacon item {0}, ' \ + 'it is in pillar.'.format(name) complete = False else: self._update_enabled(name, True) - comment = ('Enabling beacon item %s', name) + comment = 'Enabling beacon item {0}'.format(name) complete = True # Fire the complete event back along with updated list of beacons @@ -387,13 +415,13 @@ class Beacon(object): Disable a beacon ''' - if name in self.opts.get('pillar', {}).get('beacons', {}): - comment = ('Cannot disable beacon item %s, ' - 'it is in pillar.', name) + if name in self._get_beacons(include_opts=False): + comment = 'Cannot disable beacon item {0}, ' \ + 'it is in pillar.'.format(name) complete = False else: self._update_enabled(name, False) - comment = ('Disabling beacon item %s', name) + comment = 'Disabling beacon item {0}'.format(name) complete = True # Fire the complete event back along with updated list of beacons diff --git a/salt/minion.py b/salt/minion.py index d51064dc87..96ee3cf22f 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -2063,7 +2063,7 @@ class Minion(MinionBase): func = data.get(u'func', None) name = data.get(u'name', None) beacon_data = data.get(u'beacon_data', None) - include_pillar = data.get(u'include_pillar', None) + where = data.get(u'where', None) if func == u'add': self.beacons.add_beacon(name, beacon_data) @@ -2080,7 +2080,7 @@ class Minion(MinionBase): elif func == u'disable_beacon': self.beacons.disable_beacon(name) elif func == u'list': - self.beacons.list_beacons(include_pillar) + self.beacons.list_beacons(where) elif func == u'list_available': self.beacons.list_available_beacons() elif func == u'validate_beacon': diff --git a/salt/modules/beacons.py b/salt/modules/beacons.py index f5dbcbe99f..0e3208c3dc 100644 --- a/salt/modules/beacons.py +++ b/salt/modules/beacons.py @@ -28,11 +28,13 @@ __func_alias__ = { } -def list_(return_yaml=True, include_pillar=True): +def list_(return_yaml=True, where=None): ''' List the beacons currently configured on the minion :param return_yaml: Whether to return YAML formatted output, default True + :param where: Return beacon data from opts or pillar, default is + None which will return data from opts and pillar. :return: List of currently configured Beacons. CLI Example: @@ -47,7 +49,7 @@ def list_(return_yaml=True, include_pillar=True): try: eventer = salt.utils.event.get_event('minion', opts=__opts__) res = __salt__['event.fire']({'func': 'list', - 'include_pillar': include_pillar}, + 'where': where}, 'manage_beacons') if res: event_ret = eventer.get_event(tag='/salt/minion/minion_beacons_list_complete', wait=30) @@ -341,7 +343,7 @@ def save(): ret = {'comment': [], 'result': True} - beacons = list_(return_yaml=False, include_pillar=False) + beacons = list_(return_yaml=False, where='opts') # move this file into an configurable opt sfn = '{0}/{1}/beacons.conf'.format(__opts__['config_dir'], From 00063e8fc7f3f751cbbd66c48941794004217fe2 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 16 Oct 2017 14:30:37 -0700 Subject: [PATCH 064/184] Updating error messages to be more clear when beacons are configured in pillar. Updating the list to ensure the parameters are clear. --- salt/beacons/__init__.py | 28 +++++++++++++--------------- salt/minion.py | 5 +++-- salt/modules/beacons.py | 23 ++++++++++++++++------- 3 files changed, 32 insertions(+), 24 deletions(-) diff --git a/salt/beacons/__init__.py b/salt/beacons/__init__.py index 53e085f25f..4c3ec77a0d 100644 --- a/salt/beacons/__init__.py +++ b/salt/beacons/__init__.py @@ -218,21 +218,19 @@ class Beacon(object): beacons.update(opts_beacons) return beacons - def list_beacons(self, where=None): + def list_beacons(self, + include_pillar=True, + include_opts=True): ''' List the beacon items - where: Whether to include beacon data from, either opts - or pillar, default is None which would include data - from both. + include_pillar: Whether to include beacons that are + configured in pillar, default is True. + include_opts: Whether to include beacons that are + configured in opts, default is True. ''' - if where == 'pillar': - beacons = self._get_beacons(include_opts=False) - elif where == 'opts': - beacons = self._get_beacons(include_pillar=False) - else: - beacons = self._get_beacons() + beacons = self._get_beacons(include_pillar, include_opts) # Fire the complete event back along with the list of beacons evt = salt.utils.event.get_event('minion', opts=self.opts) @@ -290,7 +288,7 @@ class Beacon(object): if name in self._get_beacons(include_opts=False): comment = 'Cannot update beacon item {0}, ' \ - 'it is in pillar.'.format(name) + 'because it is configured in pillar.'.format(name) complete = False else: if name in self.opts['beacons']: @@ -319,7 +317,7 @@ class Beacon(object): if name in self._get_beacons(include_opts=False): comment = 'Cannot modify beacon item {0}, ' \ - 'it is in pillar.'.format(name) + 'it is configured in pillar.'.format(name) complete = False else: comment = 'Updating settings for beacon ' \ @@ -342,7 +340,7 @@ class Beacon(object): if name in self._get_beacons(include_opts=False): comment = 'Cannot delete beacon item {0}, ' \ - 'it is in pillar.'.format(name) + 'it is configured in pillar.'.format(name) complete = False else: if name in self.opts['beacons']: @@ -395,7 +393,7 @@ class Beacon(object): if name in self._get_beacons(include_opts=False): comment = 'Cannot enable beacon item {0}, ' \ - 'it is in pillar.'.format(name) + 'it is configured in pillar.'.format(name) complete = False else: self._update_enabled(name, True) @@ -417,7 +415,7 @@ class Beacon(object): if name in self._get_beacons(include_opts=False): comment = 'Cannot disable beacon item {0}, ' \ - 'it is in pillar.'.format(name) + 'it is configured in pillar.'.format(name) complete = False else: self._update_enabled(name, False) diff --git a/salt/minion.py b/salt/minion.py index 96ee3cf22f..dca4e1ff7c 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -2063,7 +2063,8 @@ class Minion(MinionBase): func = data.get(u'func', None) name = data.get(u'name', None) beacon_data = data.get(u'beacon_data', None) - where = data.get(u'where', None) + include_pillar = data.get(u'include_pillar', None) + include_opts = data.get(u'include_opts', None) if func == u'add': self.beacons.add_beacon(name, beacon_data) @@ -2080,7 +2081,7 @@ class Minion(MinionBase): elif func == u'disable_beacon': self.beacons.disable_beacon(name) elif func == u'list': - self.beacons.list_beacons(where) + self.beacons.list_beacons(include_opts, include_pillar) elif func == u'list_available': self.beacons.list_available_beacons() elif func == u'validate_beacon': diff --git a/salt/modules/beacons.py b/salt/modules/beacons.py index 0e3208c3dc..1aa4e745f1 100644 --- a/salt/modules/beacons.py +++ b/salt/modules/beacons.py @@ -28,14 +28,22 @@ __func_alias__ = { } -def list_(return_yaml=True, where=None): +def list_(return_yaml=True, + include_pillar=True, + include_opts=True): ''' List the beacons currently configured on the minion - :param return_yaml: Whether to return YAML formatted output, default True - :param where: Return beacon data from opts or pillar, default is - None which will return data from opts and pillar. - :return: List of currently configured Beacons. + :param return_yaml: Whether to return YAML formatted output, + default True + + :param include_pillar: Whether to include beacons that are + configured in pillar, default is True. + + :param include_opts: Whether to include beacons that are + configured in opts, default is True. + + :return: List of currently configured Beacons. CLI Example: @@ -49,7 +57,8 @@ def list_(return_yaml=True, where=None): try: eventer = salt.utils.event.get_event('minion', opts=__opts__) res = __salt__['event.fire']({'func': 'list', - 'where': where}, + 'include_pillar': include_pillar, + 'include_opts': include_opts}, 'manage_beacons') if res: event_ret = eventer.get_event(tag='/salt/minion/minion_beacons_list_complete', wait=30) @@ -343,7 +352,7 @@ def save(): ret = {'comment': [], 'result': True} - beacons = list_(return_yaml=False, where='opts') + beacons = list_(return_yaml=False, include_pillar=False) # move this file into an configurable opt sfn = '{0}/{1}/beacons.conf'.format(__opts__['config_dir'], From 2ba749e8418255275d4af3a3d238cb9ad9466b73 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 16 Oct 2017 14:38:32 -0700 Subject: [PATCH 065/184] Updating some formatting. --- salt/beacons/__init__.py | 4 ++-- salt/modules/beacons.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/beacons/__init__.py b/salt/beacons/__init__.py index 4c3ec77a0d..84bda4681d 100644 --- a/salt/beacons/__init__.py +++ b/salt/beacons/__init__.py @@ -295,7 +295,7 @@ class Beacon(object): comment = 'Updating settings for beacon ' \ 'item: {0}'.format(name) else: - comment = 'Added new beacon item {0}'.format(name) + comment = 'Added new beacon item: {0}'.format(name) complete = True self.opts['beacons'].update(data) @@ -345,7 +345,7 @@ class Beacon(object): else: if name in self.opts['beacons']: del self.opts['beacons'][name] - comment = 'Deleting beacon item {0}'.format(name) + comment = 'Deleting beacon item: {0}'.format(name) else: comment = 'Beacon item {0} not found.'.format(name) complete = True diff --git a/salt/modules/beacons.py b/salt/modules/beacons.py index 1aa4e745f1..efd1f5c939 100644 --- a/salt/modules/beacons.py +++ b/salt/modules/beacons.py @@ -147,7 +147,7 @@ def add(name, beacon_data, **kwargs): return ret if name not in list_available(return_yaml=False): - ret['comment'] = 'Beacon {0} is not available.'.format(name) + ret['comment'] = 'Beacon "{0}" is not available.'.format(name) return ret if 'test' in kwargs and kwargs['test']: From 8d1c1e21f060b9c1a0d859321449c5d1842db5a7 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 16 Oct 2017 16:43:10 -0600 Subject: [PATCH 066/184] Fix typos in paralell states docs --- doc/ref/states/parallel.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/ref/states/parallel.rst b/doc/ref/states/parallel.rst index 8a69eba2df..9edf1750e4 100644 --- a/doc/ref/states/parallel.rst +++ b/doc/ref/states/parallel.rst @@ -6,7 +6,7 @@ Introduced in Salt version ``2017.7.0`` it is now possible to run select states in parallel. This is accomplished very easily by adding the ``parallel: True`` option to your state declaration: -.. code_block:: yaml +.. code-block:: yaml nginx: service.running: @@ -24,7 +24,7 @@ state to finish. Given this example: -.. code_block:: yaml +.. code-block:: yaml sleep 10: cmd.run: @@ -74,16 +74,16 @@ also complete. Things to be Careful of ======================= -Parallel States does not prevent you from creating parallel conflicts on your +Parallel States do not prevent you from creating parallel conflicts on your system. This means that if you start multiple package installs using Salt then the package manager will block or fail. If you attempt to manage the same file with multiple states in parallel then the result can produce an unexpected file. Make sure that the states you choose to run in parallel do not conflict, or -else, like in and parallel programming environment, the outcome may not be +else, like in any parallel programming environment, the outcome may not be what you expect. Doing things like just making all states run in parallel -will almost certinly result in unexpected behavior. +will almost certainly result in unexpected behavior. With that said, running states in parallel should be safe the vast majority of the time and the most likely culprit for unexpected behavior is running From 9557504b758406bd1ac7182e92dc50866c65ad02 Mon Sep 17 00:00:00 2001 From: Tim Freund Date: Mon, 16 Oct 2017 19:26:43 -0400 Subject: [PATCH 067/184] Insert missing verb in gitfs walkthrough --- doc/topics/tutorials/gitfs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/topics/tutorials/gitfs.rst b/doc/topics/tutorials/gitfs.rst index 86dfae879d..02a7f0ecad 100644 --- a/doc/topics/tutorials/gitfs.rst +++ b/doc/topics/tutorials/gitfs.rst @@ -27,7 +27,7 @@ Installing Dependencies ======================= Both pygit2_ and GitPython_ are supported Python interfaces to git. If -compatible versions of both are installed, pygit2_ will preferred. In these +compatible versions of both are installed, pygit2_ will be preferred. In these cases, GitPython_ can be forced using the :conf_master:`gitfs_provider` parameter in the master config file. From c4d9684a904711b802f75f417dcbd6764fbaf496 Mon Sep 17 00:00:00 2001 From: Marc Koderer Date: Thu, 12 Oct 2017 10:32:45 +0200 Subject: [PATCH 068/184] Use correct mac prefix for kvm/qemu Using the private mac range for vms makes it impossible to distinguish between hypervisors. Issues #44056 Signed-off-by: Marc Koderer --- salt/modules/virt.py | 10 +++++++--- tests/unit/modules/test_virt.py | 2 ++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/salt/modules/virt.py b/salt/modules/virt.py index 2d36b5b2d2..65a89a68e2 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -656,7 +656,7 @@ def _nic_profile(profile_name, hypervisor, **kwargs): if key not in attributes or not attributes[key]: attributes[key] = value - def _assign_mac(attributes): + def _assign_mac(attributes, hypervisor): dmac = kwargs.get('dmac', None) if dmac is not None: log.debug('DMAC address is {0}'.format(dmac)) @@ -666,11 +666,15 @@ def _nic_profile(profile_name, hypervisor, **kwargs): msg = 'Malformed MAC address: {0}'.format(dmac) raise CommandExecutionError(msg) else: - attributes['mac'] = salt.utils.network.gen_mac() + if hypervisor in ['qemu', 'kvm']: + attributes['mac'] = salt.utils.network.gen_mac( + prefix='52:54:00') + else: + attributes['mac'] = salt.utils.network.gen_mac() for interface in interfaces: _normalize_net_types(interface) - _assign_mac(interface) + _assign_mac(interface, hypervisor) if hypervisor in overlays: _apply_default_overlay(interface) diff --git a/tests/unit/modules/test_virt.py b/tests/unit/modules/test_virt.py index d083b7b1e5..029bff098e 100644 --- a/tests/unit/modules/test_virt.py +++ b/tests/unit/modules/test_virt.py @@ -428,6 +428,8 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): controllers = root.findall('.//devices/controller') # There should be no controller self.assertTrue(len(controllers) == 0) + # kvm mac address shoud start with 52:54:00 + self.assertTrue("mac address='52:54:00" in xml_data) def test_mixed_dict_and_list_as_profile_objects(self): From 4c02f6ccd88eefe934cffb414b520d65e369aa83 Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Tue, 17 Oct 2017 17:03:47 +0200 Subject: [PATCH 069/184] parted: Cast boundary to string when checking unit type It could be that just a number (integer) is passed to as either start or end to the module which is then already a int. The endswith() method will then fail because that is a String function. By casting it to a String while checking we make sure that the function works. Signed-off-by: Wido den Hollander --- salt/modules/parted.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/parted.py b/salt/modules/parted.py index 72a6d3ce65..d2351e6683 100644 --- a/salt/modules/parted.py +++ b/salt/modules/parted.py @@ -93,7 +93,7 @@ def _validate_partition_boundary(boundary): ''' try: for unit in VALID_UNITS: - if boundary.endswith(unit): + if str(boundary).endswith(unit): return int(boundary) except Exception: From 74acaad46ea4c5ef09655dc5ce68e3bc6954fdbc Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 22 Sep 2017 14:34:06 -0400 Subject: [PATCH 070/184] Refactor publish func in master/masterapi to use check_authentication In a previous PR, the runner and wheel functions in master.py and masterapi.py were refactored to reduce common code via the salt.auth.check_authentication method. The publish function also can utilize the check_authentication function in master.py and masterapi.py. Consolidation of this code will help us be able to differentiate between authorization and authentication errors in the future. --- salt/auth/__init__.py | 26 ++++-- salt/daemons/masterapi.py | 121 ++++++++++----------------- salt/master.py | 121 ++++++++++----------------- tests/unit/daemons/test_masterapi.py | 111 +++++++++++++++++++++++- tests/unit/test_master.py | 106 +++++++++++++++++++++++ 5 files changed, 319 insertions(+), 166 deletions(-) diff --git a/salt/auth/__init__.py b/salt/auth/__init__.py index 6e0da617c2..483a815f5d 100644 --- a/salt/auth/__init__.py +++ b/salt/auth/__init__.py @@ -6,8 +6,6 @@ This system allows for authentication to be managed in a module pluggable way so that any external authentication system can be used inside of Salt ''' -from __future__ import absolute_import - # 1. Create auth loader instance # 2. Accept arguments as a dict # 3. Verify with function introspection @@ -16,7 +14,7 @@ from __future__ import absolute_import # 6. Interface to verify tokens # Import python libs -from __future__ import print_function +from __future__ import absolute_import, print_function import collections import time import logging @@ -31,6 +29,7 @@ import salt.transport.client import salt.utils.args import salt.utils.dictupdate import salt.utils.files +import salt.utils.master import salt.utils.minions import salt.utils.user import salt.utils.versions @@ -430,13 +429,26 @@ class LoadAuth(object): auth_list = self.get_auth_list(load) elif auth_type == 'user': - if not self.authenticate_key(load, key): + auth_ret = self.authenticate_key(load, key) + msg = 'Authentication failure of type "user" occurred' + if not auth_ret: # auth_ret can be a boolean or the effective user id if show_username: - msg = 'Authentication failure of type "user" occurred for user {0}.'.format(username) - else: - msg = 'Authentication failure of type "user" occurred' + msg = '{0} for user {1}.'.format(msg, username) ret['error'] = {'name': 'UserAuthenticationError', 'message': msg} return ret + + # Verify that the caller has root on master + if auth_ret is not True: + if AuthUser(load['user']).is_sudo(): + if not self.opts['sudo_acl'] or not self.opts['publisher_acl']: + auth_ret = True + + if auth_ret is not True: + auth_list = salt.utils.master.get_values_of_matching_keys( + self.opts['publisher_acl'], auth_ret) + if not auth_list: + ret['error'] = {'name': 'UserAuthenticationError', 'message': msg} + return ret else: ret['error'] = {'name': 'SaltInvocationError', 'message': 'Authentication type not supported.'} diff --git a/salt/daemons/masterapi.py b/salt/daemons/masterapi.py index 4bd3f08a14..5af93cbc04 100644 --- a/salt/daemons/masterapi.py +++ b/salt/daemons/masterapi.py @@ -14,6 +14,7 @@ import time import stat # Import salt libs +import salt.acl import salt.crypt import salt.cache import salt.client @@ -1176,88 +1177,50 @@ class LocalFuncs(object): ) minions = _res['minions'] - # Check for external auth calls - if extra.get('token', False): - # Authenticate - token = self.loadauth.authenticate_token(extra) - if not token: - return '' - - # Get acl from eauth module. - auth_list = self.loadauth.get_auth_list(extra, token) - - # Authorize the request - if not self.ckminions.auth_check( - auth_list, - load['fun'], - load['arg'], - load['tgt'], - load.get('tgt_type', 'glob'), - minions=minions, - # always accept find_job - whitelist=['saltutil.find_job'], - ): - log.warning('Authentication failure of type "token" occurred.') - return '' - load['user'] = token['name'] - log.debug('Minion tokenized user = "{0}"'.format(load['user'])) - elif 'eauth' in extra: - # Authenticate. - if not self.loadauth.authenticate_eauth(extra): - return '' - - # Get acl from eauth module. - auth_list = self.loadauth.get_auth_list(extra) - - # Authorize the request - if not self.ckminions.auth_check( - auth_list, - load['fun'], - load['arg'], - load['tgt'], - load.get('tgt_type', 'glob'), - minions=minions, - # always accept find_job - whitelist=['saltutil.find_job'], - ): - log.warning('Authentication failure of type "eauth" occurred.') - return '' - load['user'] = self.loadauth.load_name(extra) # The username we are attempting to auth with - # Verify that the caller has root on master + # Check for external auth calls and authenticate + auth_type, err_name, key = self._prep_auth_info(extra) + if auth_type == 'user': + auth_check = self.loadauth.check_authentication(load, auth_type, key=key) else: - auth_ret = self.loadauth.authenticate_key(load, self.key) - if auth_ret is False: + auth_check = self.loadauth.check_authentication(extra, auth_type) + + # Setup authorization list variable and error information + auth_list = auth_check.get('auth_list', []) + error = auth_check.get('error') + err_msg = 'Authentication failure of type "{0}" occurred.'.format(auth_type) + + if error: + # Authentication error occurred: do not continue. + log.warning(err_msg) + return '' + + # All Token, Eauth, and non-root users must pass the authorization check + if auth_type != 'user' or (auth_type == 'user' and auth_list): + # Authorize the request + authorized = self.ckminions.auth_check( + auth_list, + load['fun'], + load['arg'], + load['tgt'], + load.get('tgt_type', 'glob'), + minions=minions, + # always accept find_job + whitelist=['saltutil.find_job'], + ) + + if not authorized: + # Authorization error occurred. Log warning and do not continue. + log.warning(err_msg) return '' - if auth_ret is not True: - if salt.auth.AuthUser(load['user']).is_sudo(): - if not self.opts['sudo_acl'] or not self.opts['publisher_acl']: - auth_ret = True - - if auth_ret is not True: - # Avoid circular import - import salt.utils.master - auth_list = salt.utils.master.get_values_of_matching_keys( - self.opts['publisher_acl'], - auth_ret) - if not auth_list: - log.warning( - 'Authentication failure of type "user" occurred.' - ) - return '' - - if not self.ckminions.auth_check( - auth_list, - load['fun'], - load['arg'], - load['tgt'], - load.get('tgt_type', 'glob'), - minions=minions, - # always accept find_job - whitelist=['saltutil.find_job'], - ): - log.warning('Authentication failure of type "user" occurred.') - return '' + # Perform some specific auth_type tasks after the authorization check + if auth_type == 'token': + username = auth_check.get('username') + load['user'] = username + log.debug('Minion tokenized user = "{0}"'.format(username)) + elif auth_type == 'eauth': + # The username we are attempting to auth with + load['user'] = self.loadauth.load_name(extra) # If we order masters (via a syndic), don't short circuit if no minions # are found diff --git a/salt/master.py b/salt/master.py index c79dfccca0..ff8ed73426 100644 --- a/salt/master.py +++ b/salt/master.py @@ -1840,89 +1840,52 @@ class ClearFuncs(object): clear_load.get(u'tgt_type', u'glob'), delimiter ) - minions = _res.get('minions', list()) - missing = _res.get('missing', list()) + minions = _res.get(u'minions', list()) + missing = _res.get(u'missing', list()) - # Check for external auth calls - if extra.get(u'token', False): - # Authenticate. - token = self.loadauth.authenticate_token(extra) - if not token: - return u'' - - # Get acl - auth_list = self.loadauth.get_auth_list(extra, token) - - # Authorize the request - if not self.ckminions.auth_check( - auth_list, - clear_load[u'fun'], - clear_load[u'arg'], - clear_load[u'tgt'], - clear_load.get(u'tgt_type', u'glob'), - minions=minions, - # always accept find_job - whitelist=[u'saltutil.find_job'], - ): - log.warning(u'Authentication failure of type "token" occurred.') - return u'' - clear_load[u'user'] = token[u'name'] - log.debug(u'Minion tokenized user = "%s"', clear_load[u'user']) - elif u'eauth' in extra: - # Authenticate. - if not self.loadauth.authenticate_eauth(extra): - return u'' - - # Get acl from eauth module. - auth_list = self.loadauth.get_auth_list(extra) - - # Authorize the request - if not self.ckminions.auth_check( - auth_list, - clear_load[u'fun'], - clear_load[u'arg'], - clear_load[u'tgt'], - clear_load.get(u'tgt_type', u'glob'), - minions=minions, - # always accept find_job - whitelist=[u'saltutil.find_job'], - ): - log.warning(u'Authentication failure of type "eauth" occurred.') - return u'' - clear_load[u'user'] = self.loadauth.load_name(extra) # The username we are attempting to auth with - # Verify that the caller has root on master + # Check for external auth calls and authenticate + auth_type, err_name, key, sensitive_load_keys = self._prep_auth_info(extra) + if auth_type == 'user': + auth_check = self.loadauth.check_authentication(clear_load, auth_type, key=key) else: - auth_ret = self.loadauth.authenticate_key(clear_load, self.key) - if auth_ret is False: + auth_check = self.loadauth.check_authentication(extra, auth_type) + + # Setup authorization list variable and error information + auth_list = auth_check.get(u'auth_list', []) + err_msg = u'Authentication failure of type "{0}" occurred.'.format(auth_type) + + if auth_check.get(u'error'): + # Authentication error occurred: do not continue. + log.warning(err_msg) + return u'' + + # All Token, Eauth, and non-root users must pass the authorization check + if auth_type != u'user' or (auth_type == u'user' and auth_list): + # Authorize the request + authorized = self.ckminions.auth_check( + auth_list, + clear_load[u'fun'], + clear_load[u'arg'], + clear_load[u'tgt'], + clear_load.get(u'tgt_type', u'glob'), + minions=minions, + # always accept find_job + whitelist=[u'saltutil.find_job'], + ) + + if not authorized: + # Authorization error occurred. Do not continue. + log.warning(err_msg) return u'' - if auth_ret is not True: - if salt.auth.AuthUser(clear_load[u'user']).is_sudo(): - if not self.opts[u'sudo_acl'] or not self.opts[u'publisher_acl']: - auth_ret = True - - if auth_ret is not True: - auth_list = salt.utils.master.get_values_of_matching_keys( - self.opts[u'publisher_acl'], - auth_ret) - if not auth_list: - log.warning( - u'Authentication failure of type "user" occurred.' - ) - return u'' - - if not self.ckminions.auth_check( - auth_list, - clear_load[u'fun'], - clear_load[u'arg'], - clear_load[u'tgt'], - clear_load.get(u'tgt_type', u'glob'), - minions=minions, - # always accept find_job - whitelist=[u'saltutil.find_job'], - ): - log.warning(u'Authentication failure of type "user" occurred.') - return u'' + # Perform some specific auth_type tasks after the authorization check + if auth_type == u'token': + username = auth_check.get(u'username') + clear_load[u'user'] = username + log.debug(u'Minion tokenized user = "%s"', username) + elif auth_type == u'eauth': + # The username we are attempting to auth with + clear_load[u'user'] = self.loadauth.load_name(extra) # If we order masters (via a syndic), don't short circuit if no minions # are found diff --git a/tests/unit/daemons/test_masterapi.py b/tests/unit/daemons/test_masterapi.py index d2f5931227..b584d7a730 100644 --- a/tests/unit/daemons/test_masterapi.py +++ b/tests/unit/daemons/test_masterapi.py @@ -8,13 +8,16 @@ import salt.config import salt.daemons.masterapi as masterapi # Import Salt Testing Libs -from tests.support.unit import TestCase +from tests.support.unit import TestCase, skipIf from tests.support.mock import ( patch, MagicMock, + NO_MOCK, + NO_MOCK_REASON ) +@skipIf(NO_MOCK, NO_MOCK_REASON) class LocalFuncsTestCase(TestCase): ''' TestCase for salt.daemons.masterapi.LocalFuncs class @@ -24,6 +27,8 @@ class LocalFuncsTestCase(TestCase): opts = salt.config.master_config(None) self.local_funcs = masterapi.LocalFuncs(opts, 'test-key') + # runner tests + def test_runner_token_not_authenticated(self): ''' Asserts that a TokenAuthenticationError is returned when the token can't authenticate. @@ -107,6 +112,8 @@ class LocalFuncsTestCase(TestCase): self.assertDictEqual(mock_ret, ret) + # wheel tests + def test_wheel_token_not_authenticated(self): ''' Asserts that a TokenAuthenticationError is returned when the token can't authenticate. @@ -199,3 +206,105 @@ class LocalFuncsTestCase(TestCase): u'user UNKNOWN.'}} ret = self.local_funcs.wheel({}) self.assertDictEqual(mock_ret, ret) + + # publish tests + + def test_publish_user_is_blacklisted(self): + ''' + Asserts that an empty string is returned when the user has been blacklisted. + ''' + with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=True)): + self.assertEqual(u'', self.local_funcs.publish({u'user': u'foo', u'fun': u'test.arg'})) + + def test_publish_cmd_blacklisted(self): + ''' + Asserts that an empty string returned when the command has been blacklisted. + ''' + with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=True)): + self.assertEqual(u'', self.local_funcs.publish({u'user': u'foo', u'fun': u'test.arg'})) + + def test_publish_token_not_authenticated(self): + ''' + Asserts that an empty string is returned when the token can't authenticate. + ''' + load = {u'user': u'foo', u'fun': u'test.arg', u'tgt': u'test_minion', + u'kwargs': {u'token': u'asdfasdfasdfasdf'}} + with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)): + self.assertEqual(u'', self.local_funcs.publish(load)) + + def test_publish_token_authorization_error(self): + ''' + Asserts that an empty string is returned when the token authenticates, but is not + authorized. + ''' + token = u'asdfasdfasdfasdf' + load = {u'user': u'foo', u'fun': u'test.arg', u'tgt': u'test_minion', + u'arg': u'bar', u'kwargs': {u'token': token}} + mock_token = {u'token': token, u'eauth': u'foo', u'name': u'test'} + + with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.auth.LoadAuth.authenticate_token', MagicMock(return_value=mock_token)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + self.assertEqual(u'', self.local_funcs.publish(load)) + + def test_publish_eauth_not_authenticated(self): + ''' + Asserts that an empty string is returned when the user can't authenticate. + ''' + load = {u'user': u'test', u'fun': u'test.arg', u'tgt': u'test_minion', + u'kwargs': {u'eauth': u'foo'}} + with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)): + self.assertEqual(u'', self.local_funcs.publish(load)) + + def test_publish_eauth_authorization_error(self): + ''' + Asserts that an empty string is returned when the user authenticates, but is not + authorized. + ''' + load = {u'user': u'test', u'fun': u'test.arg', u'tgt': u'test_minion', + u'kwargs': {u'eauth': u'foo'}, u'arg': u'bar'} + with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.auth.LoadAuth.authenticate_eauth', MagicMock(return_value=True)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + self.assertEqual(u'', self.local_funcs.publish(load)) + + def test_publish_user_not_authenticated(self): + ''' + Asserts that an empty string is returned when the user can't authenticate. + ''' + load = {u'user': u'test', u'fun': u'test.arg', u'tgt': u'test_minion'} + with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)): + self.assertEqual(u'', self.local_funcs.publish(load)) + + def test_publish_user_authenticated_missing_auth_list(self): + ''' + Asserts that an empty string is returned when the user has an effective user id and is + authenticated, but the auth_list is empty. + ''' + load = {u'user': u'test', u'fun': u'test.arg', u'tgt': u'test_minion', + u'kwargs': {u'user': u'test'}, u'arg': u'foo'} + with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.auth.LoadAuth.authenticate_key', MagicMock(return_value='fake-user-key')), \ + patch('salt.utils.master.get_values_of_matching_keys', MagicMock(return_value=[])): + self.assertEqual(u'', self.local_funcs.publish(load)) + + def test_publish_user_authorization_error(self): + ''' + Asserts that an empty string is returned when the user authenticates, but is not + authorized. + ''' + load = {u'user': u'test', u'fun': u'test.arg', u'tgt': u'test_minion', + u'kwargs': {u'user': u'test'}, u'arg': u'foo'} + with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.auth.LoadAuth.authenticate_key', MagicMock(return_value='fake-user-key')), \ + patch('salt.utils.master.get_values_of_matching_keys', MagicMock(return_value=['test'])), \ + patch('salt.utils.minions.CkMinions.auth_check', MagicMock(return_value=False)): + self.assertEqual(u'', self.local_funcs.publish(load)) diff --git a/tests/unit/test_master.py b/tests/unit/test_master.py index b12fcb6a93..074e337983 100644 --- a/tests/unit/test_master.py +++ b/tests/unit/test_master.py @@ -24,6 +24,8 @@ class ClearFuncsTestCase(TestCase): opts = salt.config.master_config(None) self.clear_funcs = salt.master.ClearFuncs(opts, {}) + # runner tests + def test_runner_token_not_authenticated(self): ''' Asserts that a TokenAuthenticationError is returned when the token can't authenticate. @@ -116,6 +118,8 @@ class ClearFuncsTestCase(TestCase): ret = self.clear_funcs.runner({}) self.assertDictEqual(mock_ret, ret) + # wheel tests + def test_wheel_token_not_authenticated(self): ''' Asserts that a TokenAuthenticationError is returned when the token can't authenticate. @@ -207,3 +211,105 @@ class ClearFuncsTestCase(TestCase): u'message': u'Authentication failure of type "user" occurred'}} ret = self.clear_funcs.wheel({}) self.assertDictEqual(mock_ret, ret) + + # publish tests + + def test_publish_user_is_blacklisted(self): + ''' + Asserts that an empty string is returned when the user has been blacklisted. + ''' + with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=True)): + self.assertEqual(u'', self.clear_funcs.publish({u'user': u'foo', u'fun': u'test.arg'})) + + def test_publish_cmd_blacklisted(self): + ''' + Asserts that an empty string returned when the command has been blacklisted. + ''' + with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=True)): + self.assertEqual(u'', self.clear_funcs.publish({u'user': u'foo', u'fun': u'test.arg'})) + + def test_publish_token_not_authenticated(self): + ''' + Asserts that an empty string is returned when the token can't authenticate. + ''' + load = {u'user': u'foo', u'fun': u'test.arg', u'tgt': u'test_minion', + u'kwargs': {u'token': u'asdfasdfasdfasdf'}} + with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)): + self.assertEqual(u'', self.clear_funcs.publish(load)) + + def test_publish_token_authorization_error(self): + ''' + Asserts that an empty string is returned when the token authenticates, but is not + authorized. + ''' + token = u'asdfasdfasdfasdf' + load = {u'user': u'foo', u'fun': u'test.arg', u'tgt': u'test_minion', + u'arg': u'bar', u'kwargs': {u'token': token}} + mock_token = {u'token': token, u'eauth': u'foo', u'name': u'test'} + + with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.auth.LoadAuth.authenticate_token', MagicMock(return_value=mock_token)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + self.assertEqual(u'', self.clear_funcs.publish(load)) + + def test_publish_eauth_not_authenticated(self): + ''' + Asserts that an empty string is returned when the user can't authenticate. + ''' + load = {u'user': u'test', u'fun': u'test.arg', u'tgt': u'test_minion', + u'kwargs': {u'eauth': u'foo'}} + with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)): + self.assertEqual(u'', self.clear_funcs.publish(load)) + + def test_publish_eauth_authorization_error(self): + ''' + Asserts that an empty string is returned when the user authenticates, but is not + authorized. + ''' + load = {u'user': u'test', u'fun': u'test.arg', u'tgt': u'test_minion', + u'kwargs': {u'eauth': u'foo'}, u'arg': u'bar'} + with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.auth.LoadAuth.authenticate_eauth', MagicMock(return_value=True)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + self.assertEqual(u'', self.clear_funcs.publish(load)) + + def test_publish_user_not_authenticated(self): + ''' + Asserts that an empty string is returned when the user can't authenticate. + ''' + load = {u'user': u'test', u'fun': u'test.arg', u'tgt': u'test_minion'} + with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)): + self.assertEqual(u'', self.clear_funcs.publish(load)) + + def test_publish_user_authenticated_missing_auth_list(self): + ''' + Asserts that an empty string is returned when the user has an effective user id and is + authenticated, but the auth_list is empty. + ''' + load = {u'user': u'test', u'fun': u'test.arg', u'tgt': u'test_minion', + u'kwargs': {u'user': u'test'}, u'arg': u'foo'} + with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.auth.LoadAuth.authenticate_key', MagicMock(return_value='fake-user-key')), \ + patch('salt.utils.master.get_values_of_matching_keys', MagicMock(return_value=[])): + self.assertEqual(u'', self.clear_funcs.publish(load)) + + def test_publish_user_authorization_error(self): + ''' + Asserts that an empty string is returned when the user authenticates, but is not + authorized. + ''' + load = {u'user': u'test', u'fun': u'test.arg', u'tgt': u'test_minion', + u'kwargs': {u'user': u'test'}, u'arg': u'foo'} + with patch('salt.acl.PublisherACL.user_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.acl.PublisherACL.cmd_is_blacklisted', MagicMock(return_value=False)), \ + patch('salt.auth.LoadAuth.authenticate_key', MagicMock(return_value='fake-user-key')), \ + patch('salt.utils.master.get_values_of_matching_keys', MagicMock(return_value=['test'])), \ + patch('salt.utils.minions.CkMinions.auth_check', MagicMock(return_value=False)): + self.assertEqual(u'', self.clear_funcs.publish(load)) From 6823cdcbfaa3e391ef227d1533507e3cf53ce863 Mon Sep 17 00:00:00 2001 From: Ben Harper Date: Tue, 17 Oct 2017 19:21:29 +0000 Subject: [PATCH 071/184] add documentation for venv_bin in virtualenv_mod state --- salt/states/virtualenv_mod.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/states/virtualenv_mod.py b/salt/states/virtualenv_mod.py index bd394bfd0b..2df058a7ce 100644 --- a/salt/states/virtualenv_mod.py +++ b/salt/states/virtualenv_mod.py @@ -67,6 +67,10 @@ def managed(name, name Path to the virtualenv. + venv_bin: virtualenv + The name (and optionally path) of the virtualenv command. This can also + be set globally in the minion config file as ``virtualenv.venv_bin``. + requirements: None Path to a pip requirements file. If the path begins with ``salt://`` the file will be transferred from the master file server. From b615ce1762ee5611298d87f3b85718b106b55043 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Thu, 5 Oct 2017 10:08:20 -0600 Subject: [PATCH 072/184] if expect_minions is passed use that instead In the salt/states/saltmod.py, we set expect minions sometimes and pass that through. In this instance, we should use the expect_minions that is passed so that it does not get passed along twice. --- salt/client/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/client/__init__.py b/salt/client/__init__.py index 9ec5fc3666..062ed5fc76 100644 --- a/salt/client/__init__.py +++ b/salt/client/__init__.py @@ -1582,7 +1582,7 @@ class LocalClient(object): timeout=timeout, tgt=tgt, tgt_type=tgt_type, - expect_minions=(verbose or show_timeout), + expect_minions=(kwargs.pop('expect_minions', False) or verbose or show_timeout), **kwargs ): log.debug('return event: %s', ret) From 272dcc6ba5a9bdaab08e5e44b46eb1929484055c Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 17 Oct 2017 14:57:33 -0600 Subject: [PATCH 073/184] add inline comment about popping expect_minions --- salt/client/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/client/__init__.py b/salt/client/__init__.py index 062ed5fc76..604fc94677 100644 --- a/salt/client/__init__.py +++ b/salt/client/__init__.py @@ -1582,6 +1582,9 @@ class LocalClient(object): timeout=timeout, tgt=tgt, tgt_type=tgt_type, + # (gtmanfred) expect_minions is popped here incase it is passed from a client + # call. If this is not popped, then it would be passed twice to + # get_iter_returns. expect_minions=(kwargs.pop('expect_minions', False) or verbose or show_timeout), **kwargs ): From 1e1e5a3ff682e348ec389c0b33c32b54ff4ee807 Mon Sep 17 00:00:00 2001 From: Andreas Lutro Date: Sat, 14 Oct 2017 20:01:41 +0200 Subject: [PATCH 074/184] try to correctly parse debian codename from /etc/os-release --- salt/grains/core.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index 0a98bc148f..48481fd3f8 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -1367,7 +1367,10 @@ def os_data(): .format(' '.join(init_cmdline)) ) - # Add lsb grains on any distro with lsb-release + # Add lsb grains on any distro with lsb-release. Note that this import + # can fail on systems with lsb-release installed if the system package + # does not install the python package for the python interpreter used by + # Salt (i.e. python2 or python3) try: import lsb_release # pylint: disable=import-error release = lsb_release.get_distro_information() @@ -1416,7 +1419,13 @@ def os_data(): if 'VERSION_ID' in os_release: grains['lsb_distrib_release'] = os_release['VERSION_ID'] if 'PRETTY_NAME' in os_release: - grains['lsb_distrib_codename'] = os_release['PRETTY_NAME'] + codename = os_release['PRETTY_NAME'] + # https://github.com/saltstack/salt/issues/44108 + if os_release['ID'] == 'debian': + codename_match = re.search(r'\((\w+)\)$', codename) + if codename_match: + codename = codename_match.group(1) + grains['lsb_distrib_codename'] = codename if 'CPE_NAME' in os_release: if ":suse:" in os_release['CPE_NAME'] or ":opensuse:" in os_release['CPE_NAME']: grains['os'] = "SUSE" From 0311c2a4dee1c303641baa6953068383dba952d1 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 17 Oct 2017 17:55:18 -0400 Subject: [PATCH 075/184] Replace salt utils import with correct salt.utils.files path This fixes a couple of test failures in the develop branch and cleans up a couple of formatting issues and import ordering. --- tests/integration/modules/test_groupadd.py | 30 +++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/integration/modules/test_groupadd.py b/tests/integration/modules/test_groupadd.py index 9963793ca1..9936fc7411 100644 --- a/tests/integration/modules/test_groupadd.py +++ b/tests/integration/modules/test_groupadd.py @@ -2,18 +2,18 @@ # Import python libs from __future__ import absolute_import -import string +import grp +import os import random +import string # Import Salt Testing libs from tests.support.case import ModuleCase from tests.support.helpers import destructiveTest, skip_if_not_root -# Import 3rd-party libs +# Import Salt libs from salt.ext.six.moves import range -import os -import grp -from salt import utils +import salt.utils.files @skip_if_not_root @@ -66,7 +66,7 @@ class GroupModuleTest(ModuleCase): ''' defs_file = '/etc/login.defs' if os.path.exists(defs_file): - with utils.fopen(defs_file) as defs_fd: + with salt.utils.files.fopen(defs_file) as defs_fd: login_defs = dict([x.split() for x in defs_fd.readlines() if x.strip() @@ -102,12 +102,12 @@ class GroupModuleTest(ModuleCase): ''' Test the add group function ''' - #add a new group + # add a new group self.assertTrue(self.run_function('group.add', [self._group, self._gid])) group_info = self.run_function('group.info', [self._group]) self.assertEqual(group_info['name'], self._group) self.assertEqual(group_info['gid'], self._gid) - #try adding the group again + # try adding the group again self.assertFalse(self.run_function('group.add', [self._group, self._gid])) @destructiveTest @@ -124,7 +124,7 @@ class GroupModuleTest(ModuleCase): group_info = self.run_function('group.info', [self._group]) self.assertEqual(group_info['name'], self._group) self.assertTrue(gid_min <= group_info['gid'] <= gid_max) - #try adding the group again + # try adding the group again self.assertFalse(self.run_function('group.add', [self._group])) @@ -142,7 +142,7 @@ class GroupModuleTest(ModuleCase): group_info = self.run_function('group.info', [self._group]) self.assertEqual(group_info['name'], self._group) self.assertEqual(group_info['gid'], gid) - #try adding the group again + # try adding the group again self.assertFalse(self.run_function('group.add', [self._group, gid])) @@ -153,10 +153,10 @@ class GroupModuleTest(ModuleCase): ''' self.assertTrue(self.run_function('group.add', [self._group])) - #correct functionality + # correct functionality self.assertTrue(self.run_function('group.delete', [self._group])) - #group does not exist + # group does not exist self.assertFalse(self.run_function('group.delete', [self._no_group])) @destructiveTest @@ -193,11 +193,11 @@ class GroupModuleTest(ModuleCase): self.assertTrue(self.run_function('group.adduser', [self._group, self._user])) group_info = self.run_function('group.info', [self._group]) self.assertIn(self._user, group_info['members']) - #try add a non existing user + # try to add a non existing user self.assertFalse(self.run_function('group.adduser', [self._group, self._no_user])) - #try add a user to non existing group + # try to add a user to non existing group self.assertFalse(self.run_function('group.adduser', [self._no_group, self._user])) - #try add a non existing user to a non existing group + # try to add a non existing user to a non existing group self.assertFalse(self.run_function('group.adduser', [self._no_group, self._no_user])) @destructiveTest From f3980d7d838951bba3ef1ade655e9df6c720f6e3 Mon Sep 17 00:00:00 2001 From: Ken Crowell Date: Mon, 9 Oct 2017 12:18:25 -0300 Subject: [PATCH 076/184] Fix manage.present to show lost minions Fixes #38367 and and #43936. --- salt/utils/minions.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/salt/utils/minions.py b/salt/utils/minions.py index 261022d303..7e8f5e685f 100644 --- a/salt/utils/minions.py +++ b/salt/utils/minions.py @@ -578,10 +578,9 @@ class CkMinions(object): if search is None: return minions addrs = salt.utils.network.local_port_tcp(int(self.opts['publish_port'])) - if '127.0.0.1' in addrs or '0.0.0.0' in addrs: - # Add in possible ip addresses of a locally connected minion + if '127.0.0.1' in addrs: + # Add in the address of a possible locally-connected minion. addrs.discard('127.0.0.1') - addrs.discard('0.0.0.0') addrs.update(set(salt.utils.network.ip_addrs(include_loopback=include_localhost))) if subset: search = subset From d712031a435ea4376bcf4168c13dda53547314f4 Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 18 Oct 2017 09:44:55 -0400 Subject: [PATCH 077/184] Update to_unicode util references to new stringutils path --- tests/unit/templates/test_jinja.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unit/templates/test_jinja.py b/tests/unit/templates/test_jinja.py index 08e7e2e5f1..e5ca422f92 100644 --- a/tests/unit/templates/test_jinja.py +++ b/tests/unit/templates/test_jinja.py @@ -33,6 +33,7 @@ from salt.utils.jinja import ( ) from salt.utils.templates import JINJA, render_jinja_tmpl from salt.utils.odict import OrderedDict +import salt.utils.stringutils # Import 3rd party libs import yaml @@ -296,7 +297,7 @@ class TestGetTemplate(TestCase): 'file_roots': self.local_opts['file_roots'], 'pillar_roots': self.local_opts['pillar_roots']}, a='Hi', b='Sàlt', saltenv='test', salt=self.local_salt)) - self.assertEqual(out, salt.utils.to_unicode('Hey world !Hi Sàlt !' + os.linesep)) + self.assertEqual(out, salt.utils.stringutils.to_unicode('Hey world !Hi Sàlt !' + os.linesep)) self.assertEqual(fc.requests[0]['path'], 'salt://macro') filename = os.path.join(TEMPLATES_DIR, 'files', 'test', 'non_ascii') @@ -370,8 +371,8 @@ class TestGetTemplate(TestCase): with salt.utils.files.fopen(out['data']) as fp: result = fp.read() if six.PY2: - result = salt.utils.to_unicode(result) - self.assertEqual(salt.utils.to_unicode('Assunção' + os.linesep), result) + result = salt.utils.stringutils.to_unicode(result) + self.assertEqual(salt.utils.stringutils.to_unicode('Assunção' + os.linesep), result) def test_get_context_has_enough_context(self): template = '1\n2\n3\n4\n5\n6\n7\n8\n9\na\nb\nc\nd\ne\nf' From 33fb7331a9b51bceaf3699602f07d2dc153b3447 Mon Sep 17 00:00:00 2001 From: Vernon Cole Date: Thu, 12 Oct 2017 20:04:21 -0600 Subject: [PATCH 078/184] convert netapi calls to client calls --- salt/cloud/clouds/saltify.py | 110 +++++++++++------------------------ 1 file changed, 33 insertions(+), 77 deletions(-) diff --git a/salt/cloud/clouds/saltify.py b/salt/cloud/clouds/saltify.py index 1f3d0867bd..3215d8a32c 100644 --- a/salt/cloud/clouds/saltify.py +++ b/salt/cloud/clouds/saltify.py @@ -19,7 +19,7 @@ import logging # Import salt libs import salt.utils.cloud import salt.config as config -import salt.netapi +import salt.client import salt.ext.six as six if six.PY3: import ipaddress @@ -32,6 +32,7 @@ from salt.exceptions import SaltCloudException, SaltCloudSystemExit log = logging.getLogger(__name__) try: + # noinspection PyUnresolvedReferences from impacket.smbconnection import SessionError as smbSessionError from impacket.smb3 import SessionError as smb3SessionError HAS_IMPACKET = True @@ -39,7 +40,9 @@ except ImportError: HAS_IMPACKET = False try: + # noinspection PyUnresolvedReferences from winrm.exceptions import WinRMTransportError + # noinspection PyUnresolvedReferences from requests.exceptions import ( ConnectionError, ConnectTimeout, ReadTimeout, SSLError, ProxyError, RetryError, InvalidSchema) @@ -55,24 +58,6 @@ def __virtual__(): return True -def _get_connection_info(): - ''' - Return connection information for the passed VM data - ''' - vm_ = get_configured_provider() - - try: - ret = {'username': vm_['username'], - 'password': vm_['password'], - 'eauth': vm_['eauth'], - 'vm': vm_, - } - except KeyError: - raise SaltCloudException( - 'Configuration must define salt-api "username", "password" and "eauth"') - return ret - - def avail_locations(call=None): ''' This function returns a list of locations available. @@ -81,7 +66,7 @@ def avail_locations(call=None): salt-cloud --list-locations my-cloud-provider - [ saltify will always returns an empty dictionary ] + [ saltify will always return an empty dictionary ] ''' return {} @@ -127,8 +112,6 @@ def list_nodes(call=None): returns a list of dictionaries of defined standard fields. - salt-api setup required for operation. - ..versionadded:: Oxygen ''' @@ -172,8 +155,8 @@ def list_nodes_full(call=None): salt-cloud -F returns a list of dictionaries. + for 'saltify' minions, returns dict of grains (enhanced). - salt-api setup required for operation. ..versionadded:: Oxygen ''' @@ -200,16 +183,9 @@ def _list_nodes_full(call=None): ''' List the nodes, ask all 'saltify' minions, return dict of grains. ''' - local = salt.netapi.NetapiClient(__opts__) - cmd = {'client': 'local', - 'tgt': 'salt-cloud:driver:saltify', - 'fun': 'grains.items', - 'arg': '', - 'tgt_type': 'grain', - } - cmd.update(_get_connection_info()) - - return local.run(cmd) + local = salt.client.LocalClient() + return local.cmd('salt-cloud:driver:saltify', 'grains.items', '', + tgt_type='grain') def list_nodes_select(call=None): @@ -226,15 +202,8 @@ def show_instance(name, call=None): ''' List the a single node, return dict of grains. ''' - local = salt.netapi.NetapiClient(__opts__) - cmd = {'client': 'local', - 'tgt': 'name', - 'fun': 'grains.items', - 'arg': '', - 'tgt_type': 'glob', - } - cmd.update(_get_connection_info()) - ret = local.run(cmd) + local = salt.client.LocalClient() + ret = local.cmd(name, 'grains.items') ret.update(_build_required_items(ret)) return ret @@ -365,14 +334,21 @@ def destroy(name, call=None): .. versionadded:: Oxygen + Disconnect a minion from the master, and remove its keys. + + Optionally, (if ``remove_config_on_destroy`` is ``True``), + disables salt-minion from running on the minion, and + erases the Salt configuration files from it. + + Optionally, (if ``shutdown_on_destroy`` is ``True``), + orders the minion to halt. + CLI Example: .. code-block:: bash salt-cloud --destroy mymachine - salt-api setup required for operation. - ''' if call == 'function': raise SaltCloudSystemExit( @@ -391,15 +367,9 @@ def destroy(name, call=None): transport=opts['transport'] ) - local = salt.netapi.NetapiClient(opts) - cmd = {'client': 'local', - 'tgt': name, - 'fun': 'grains.get', - 'arg': ['salt-cloud'], - } - cmd.update(_get_connection_info()) - vm_ = cmd['vm'] - my_info = local.run(cmd) + vm_ = get_configured_provider() + local = salt.client.LocalClient() + my_info = local.cmd(name, 'grains.get', ['salt-cloud']) try: vm_.update(my_info[name]) # get profile name to get config value except (IndexError, TypeError): @@ -407,25 +377,22 @@ def destroy(name, call=None): if config.get_cloud_config_value( 'remove_config_on_destroy', vm_, opts, default=True ): - cmd.update({'fun': 'service.disable', 'arg': ['salt-minion']}) - ret = local.run(cmd) # prevent generating new keys on restart + ret = local.cmd(name, # prevent generating new keys on restart + 'service.disable', + ['salt-minion']) if ret and ret[name]: log.info('disabled salt-minion service on %s', name) - cmd.update({'fun': 'config.get', 'arg': ['conf_file']}) - ret = local.run(cmd) + ret = local.cmd(name, 'config.get', ['conf_file']) if ret and ret[name]: confile = ret[name] - cmd.update({'fun': 'file.remove', 'arg': [confile]}) - ret = local.run(cmd) + ret = local.cmd(name, 'file.remove', [confile]) if ret and ret[name]: log.info('removed minion %s configuration file %s', name, confile) - cmd.update({'fun': 'config.get', 'arg': ['pki_dir']}) - ret = local.run(cmd) + ret = local.cmd(name, 'config.get', ['pki_dir']) if ret and ret[name]: pki_dir = ret[name] - cmd.update({'fun': 'file.remove', 'arg': [pki_dir]}) - ret = local.run(cmd) + ret = local.cmd(name, 'file.remove', [pki_dir]) if ret and ret[name]: log.info( 'removed minion %s key files in %s', @@ -435,8 +402,7 @@ def destroy(name, call=None): if config.get_cloud_config_value( 'shutdown_on_destroy', vm_, opts, default=False ): - cmd.update({'fun': 'system.shutdown', 'arg': ''}) - ret = local.run(cmd) + ret = local.cmd(name, 'system.shutdown') if ret and ret[name]: log.info('system.shutdown for minion %s successful', name) @@ -456,8 +422,6 @@ def reboot(name, call=None): ''' Reboot a saltify minion. - salt-api setup required for operation. - ..versionadded:: Oxygen name @@ -475,13 +439,5 @@ def reboot(name, call=None): 'The reboot action must be called with -a or --action.' ) - local = salt.netapi.NetapiClient(__opts__) - cmd = {'client': 'local', - 'tgt': name, - 'fun': 'system.reboot', - 'arg': '', - } - cmd.update(_get_connection_info()) - ret = local.run(cmd) - - return ret + local = salt.client.LocalClient() + return local.cmd(name, 'system.reboot') From 805d5106ed1b56976994468b69af3b35b4ba841a Mon Sep 17 00:00:00 2001 From: Vernon Cole Date: Thu, 12 Oct 2017 20:12:33 -0600 Subject: [PATCH 079/184] remove references to salt-api from docs --- doc/topics/cloud/saltify.rst | 72 ++---------------------------------- 1 file changed, 3 insertions(+), 69 deletions(-) diff --git a/doc/topics/cloud/saltify.rst b/doc/topics/cloud/saltify.rst index 8b59da5b0e..4f1b41546d 100644 --- a/doc/topics/cloud/saltify.rst +++ b/doc/topics/cloud/saltify.rst @@ -33,9 +33,9 @@ the salt-master: However, if you wish to use the more advanced capabilities of salt-cloud, such as rebooting, listing, and disconnecting machines, then the salt master must fill -the role usually performed by a vendor's cloud management system. In order to do -that, you must configure your salt master as a salt-api server, and supply credentials -to use it. (See ``salt-api setup`` below.) +the role usually performed by a vendor's cloud management system. The salt master +must be running on the salt-cloud machine, and created nodes must be connected to the +master. Profiles @@ -165,69 +165,3 @@ Return values: - ``True``: Credential verification succeeded - ``False``: Credential verification succeeded - ``None``: Credential verification was not attempted. - -Provisioning salt-api -===================== - -In order to query or control minions it created, saltify needs to send commands -to the salt master. It does that using the network interface to salt-api. - -The salt-api is not enabled by default. The following example will provide a -simple installation. - -.. code-block:: yaml - - # file /etc/salt/cloud.profiles.d/my_saltify_profiles.conf - hw_41: # a theoretical example hardware machine - ssh_host: 10.100.9.41 # the hard address of your target - ssh_username: vagrant # a user name which has passwordless sudo - password: vagrant # on your target machine - provider: my_saltify_provider - shutdown_on_destroy: true # halt the target on "salt-cloud -d" command - - - -.. code-block:: yaml - - # file /etc/salt/cloud.providers.d/saltify_provider.conf - my_saltify_provider: - driver: saltify - eauth: pam - username: vagrant # supply some sudo-group-member's name - password: vagrant # and password on the salt master - minion: - master: 10.100.9.5 # the hard address of the master - - -.. code-block:: yaml - - # file /etc/salt/master.d/auth.conf - # using salt-api ... members of the 'sudo' group can do anything ... - external_auth: - pam: - sudo%: - - .* - - '@wheel' - - '@runner' - - '@jobs' - - -.. code-block:: yaml - - # file /etc/salt/master.d/api.conf - # see https://docs.saltstack.com/en/latest/ref/netapi/all/salt.netapi.rest_cherrypy.html - rest_cherrypy: - host: localhost - port: 8000 - ssl_crt: /etc/pki/tls/certs/localhost.crt - ssl_key: /etc/pki/tls/certs/localhost.key - thread_pool: 30 - socket_queue_size: 10 - - -Start your target machine as a Salt minion named "node41" by: - -.. code-block:: bash - - $ sudo salt-cloud -p hw_41 node41 - From f1d3c5bbcfe908462e20aa2a41c5efb89cc08547 Mon Sep 17 00:00:00 2001 From: Benedikt Werner <1benediktwerner@gmail.com> Date: Wed, 18 Oct 2017 17:37:57 +0200 Subject: [PATCH 080/184] Added 'versionadded' tags to sensehat modules --- salt/beacons/sensehat.py | 2 ++ salt/modules/sensehat.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/salt/beacons/sensehat.py b/salt/beacons/sensehat.py index 65f2ff134b..2f7cefebdf 100644 --- a/salt/beacons/sensehat.py +++ b/salt/beacons/sensehat.py @@ -3,6 +3,8 @@ Beacon to monitor temperature, humidity and pressure using the SenseHat of a Raspberry Pi. +.. versionadded:: 2017.7 + :maintainer: Benedikt Werner <1benediktwerner@gmail.com> :maturity: new :depends: sense_hat Python module diff --git a/salt/modules/sensehat.py b/salt/modules/sensehat.py index f857a1e74d..648a59cd29 100644 --- a/salt/modules/sensehat.py +++ b/salt/modules/sensehat.py @@ -2,6 +2,8 @@ ''' Module for controlling the LED matrix or reading environment data on the SenseHat of a Raspberry Pi. +.. versionadded:: 2017.7 + :maintainer: Benedikt Werner <1benediktwerner@gmail.com>, Joachim Werner :maturity: new :depends: sense_hat Python module From 534faf0b7ac2df0f8ff6a8892c34d6fa5fb85bdb Mon Sep 17 00:00:00 2001 From: Vasili Syrakis Date: Fri, 13 Oct 2017 22:54:58 +1100 Subject: [PATCH 081/184] Catch on empty Virtualbox network addr #43427 --- salt/utils/virtualbox.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/utils/virtualbox.py b/salt/utils/virtualbox.py index 9bcc0eab74..08190f113b 100644 --- a/salt/utils/virtualbox.py +++ b/salt/utils/virtualbox.py @@ -278,7 +278,10 @@ def vb_get_network_addresses(machine_name=None, machine=None): # We can't trust virtualbox to give us up to date guest properties if the machine isn't running # For some reason it may give us outdated (cached?) values if machine.state == _virtualboxManager.constants.MachineState_Running: - total_slots = int(machine.getGuestPropertyValue("/VirtualBox/GuestInfo/Net/Count")) + try: + total_slots = int(machine.getGuestPropertyValue('/VirtualBox/GuestInfo/Net/Count')) + except ValueError: + total_slots = 0 for i in range(total_slots): try: address = machine.getGuestPropertyValue("/VirtualBox/GuestInfo/Net/{0}/V4/IP".format(i)) From bd825b51cc1cd72e684251b51aa53d91bfe76f5f Mon Sep 17 00:00:00 2001 From: Benedikt Werner <1benediktwerner@gmail.com> Date: Wed, 18 Oct 2017 19:08:59 +0200 Subject: [PATCH 082/184] Changed sensehat versionadded from 2017.7 to 2017.7.0 --- salt/beacons/sensehat.py | 2 +- salt/modules/sensehat.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/beacons/sensehat.py b/salt/beacons/sensehat.py index 2f7cefebdf..9be6d682b1 100644 --- a/salt/beacons/sensehat.py +++ b/salt/beacons/sensehat.py @@ -3,7 +3,7 @@ Beacon to monitor temperature, humidity and pressure using the SenseHat of a Raspberry Pi. -.. versionadded:: 2017.7 +.. versionadded:: 2017.7.0 :maintainer: Benedikt Werner <1benediktwerner@gmail.com> :maturity: new diff --git a/salt/modules/sensehat.py b/salt/modules/sensehat.py index 648a59cd29..12da3b0742 100644 --- a/salt/modules/sensehat.py +++ b/salt/modules/sensehat.py @@ -2,7 +2,7 @@ ''' Module for controlling the LED matrix or reading environment data on the SenseHat of a Raspberry Pi. -.. versionadded:: 2017.7 +.. versionadded:: 2017.7.0 :maintainer: Benedikt Werner <1benediktwerner@gmail.com>, Joachim Werner :maturity: new From 77b948b00ae5f073074f28d893dfae84c6b41968 Mon Sep 17 00:00:00 2001 From: Benedikt Werner <1benediktwerner@gmail.com> Date: Wed, 18 Oct 2017 17:06:16 +0200 Subject: [PATCH 083/184] Skip shadow module unit test if not root or no shadow --- tests/unit/modules/test_shadow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/modules/test_shadow.py b/tests/unit/modules/test_shadow.py index e152d59b9a..6040b83ab9 100644 --- a/tests/unit/modules/test_shadow.py +++ b/tests/unit/modules/test_shadow.py @@ -42,6 +42,7 @@ _HASHES = dict( @skipIf(not salt.utils.platform.is_linux(), 'minion is not Linux') +@skipIf(not HAS_SHADOW, 'shadow module is not available') class LinuxShadowTest(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): @@ -62,8 +63,7 @@ class LinuxShadowTest(TestCase, LoaderModuleMockMixin): hash_info['pw_hash'] ) - # 'list_users' function tests: 1 - + @skip_if_not_root def test_list_users(self): ''' Test if it returns a list of all users From e210f12cb7d551b9e4cbbb2136381d01640ee681 Mon Sep 17 00:00:00 2001 From: kstreee Date: Tue, 17 Oct 2017 17:59:24 +0900 Subject: [PATCH 084/184] Fixes inconsistent exception handling in rest_tornado. --- salt/netapi/rest_tornado/saltnado.py | 5 +--- .../unit/netapi/rest_tornado/test_handlers.py | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py index d40edf44e0..96b133aeed 100644 --- a/salt/netapi/rest_tornado/saltnado.py +++ b/salt/netapi/rest_tornado/saltnado.py @@ -908,10 +908,7 @@ class SaltAPIHandler(BaseSaltAPIHandler, SaltClientsMixIn): # pylint: disable=W f_call = self._format_call_run_job_async(chunk) # fire a job off - try: - pub_data = yield self.saltclients['local'](*f_call.get('args', ()), **f_call.get('kwargs', {})) - except EauthAuthenticationError: - raise tornado.gen.Return('Not authorized to run this job') + pub_data = yield self.saltclients['local'](*f_call.get('args', ()), **f_call.get('kwargs', {})) # if the job didn't publish, lets not wait around for nothing # TODO: set header?? diff --git a/tests/unit/netapi/rest_tornado/test_handlers.py b/tests/unit/netapi/rest_tornado/test_handlers.py index 23bf5188ca..c7e88ccbdb 100644 --- a/tests/unit/netapi/rest_tornado/test_handlers.py +++ b/tests/unit/netapi/rest_tornado/test_handlers.py @@ -13,6 +13,7 @@ from tests.support.unit import TestCase, skipIf # Import Salt libs import salt.auth +from salt.ext.six.moves import map # pylint: disable=import-error try: import salt.netapi.rest_tornado as rest_tornado from salt.netapi.rest_tornado import saltnado @@ -619,6 +620,34 @@ class TestSaltAuthHandler(SaltnadoTestCase): self.assertEqual(response.code, 400) +class TestSaltRunHandler(SaltnadoTestCase): + + def get_app(self): + urls = [('/run', saltnado.RunSaltAPIHandler)] + return self.build_tornado_app(urls) + + def test_authentication_exception_consistency(self): + ''' + Test consistency of authentication exception of each clients. + ''' + valid_response = {'return': ['Failed to authenticate']} + + clients = ['local', 'local_async', 'runner', 'runner_async'] + request_lowstates = map(lambda client: {"client": client, + "tgt": "*", + "fun": "test.fib", + "arg": ["10"]}, + clients) + + for request_lowstate in request_lowstates: + response = self.fetch('/run', + method='POST', + body=json.dumps(request_lowstate), + headers={'Content-Type': self.content_type_map['json']}) + + self.assertEqual(valid_response, json.loads(response.body)) + + @skipIf(HAS_TORNADO is False, 'The tornado package needs to be installed') # pylint: disable=W0223 class TestWebsocketSaltAPIHandler(SaltnadoTestCase): From 832aa3010ceb229d151e2d94520dfbb0571318b7 Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 18 Oct 2017 14:15:50 -0400 Subject: [PATCH 085/184] Reorder and group imports in jina template unit tests The imports were scattered around quite a bit, which made it hard to see where multiple uses of salt/utils/* imports were being pulled in. This PR just adjusts the ordering so it's easier to see what is included already for the future. (This is particularly useful for catching misuses of salt.utils.__init__.py functions.) --- tests/unit/templates/test_jinja.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/unit/templates/test_jinja.py b/tests/unit/templates/test_jinja.py index e5ca422f92..0975dc2da7 100644 --- a/tests/unit/templates/test_jinja.py +++ b/tests/unit/templates/test_jinja.py @@ -2,14 +2,16 @@ # Import python libs from __future__ import absolute_import -import os +from jinja2 import Environment, DictLoader, exceptions import ast import copy -import tempfile -import json import datetime +import json +import os import pprint import re +import tempfile +import yaml # Import Salt Testing libs from tests.support.unit import skipIf, TestCase @@ -17,27 +19,30 @@ from tests.support.case import ModuleCase from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, MagicMock from tests.support.paths import TMP_CONF_DIR -# Import salt libs +# Import Salt libs import salt.config -from salt.ext import six import salt.loader -import salt.utils.files -from salt.utils import get_context from salt.exceptions import SaltRenderError + +from salt.ext import six from salt.ext.six.moves import builtins + from salt.utils.decorators.jinja import JinjaFilter from salt.utils.jinja import ( SaltCacheLoader, SerializerExtension, ensure_sequence_filter ) -from salt.utils.templates import JINJA, render_jinja_tmpl from salt.utils.odict import OrderedDict +from salt.utils.templates import ( + get_context, + JINJA, + render_jinja_tmpl +) +import salt.utils.files import salt.utils.stringutils # Import 3rd party libs -import yaml -from jinja2 import Environment, DictLoader, exceptions try: import timelib # pylint: disable=W0611 HAS_TIMELIB = True From 7b46489e33ac0e374841df8516041ada0533ef5d Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 18 Oct 2017 13:34:11 -0500 Subject: [PATCH 086/184] Fix examples in docker_container.{stopped,absent} docstrings These were erroneously changed from `dockerng.{stopped,absent}` to `docker.{stopped,absent}` using sed, when they should have been changed to reflect the new name of this state module (like the `docker_container.running` examples were). --- salt/states/docker_container.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/states/docker_container.py b/salt/states/docker_container.py index 1f5996c3d5..c5eb7ee597 100644 --- a/salt/states/docker_container.py +++ b/salt/states/docker_container.py @@ -1853,7 +1853,7 @@ def stopped(name=None, .. code-block:: yaml stopped_containers: - docker.stopped: + docker_container.stopped: - names: - foo - bar @@ -1862,7 +1862,7 @@ def stopped(name=None, .. code-block:: yaml stopped_containers: - docker.stopped: + docker_container.stopped: - containers: - foo - bar @@ -1998,10 +1998,10 @@ def absent(name, force=False): .. code-block:: yaml mycontainer: - docker.absent + docker_container.absent multiple_containers: - docker.absent: + docker_container.absent: - names: - foo - bar From 5f7555846f47df7f3d5e1aa4e6a01e88d3ae0a18 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 18 Oct 2017 12:51:29 -0700 Subject: [PATCH 087/184] When looping through the various pre, post, up and down commands put them into the interface dict using the right internet family variable. --- salt/modules/debian_ip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/debian_ip.py b/salt/modules/debian_ip.py index 025fe2d4d7..55269e8f71 100644 --- a/salt/modules/debian_ip.py +++ b/salt/modules/debian_ip.py @@ -1388,7 +1388,7 @@ def _parse_settings_eth(opts, iface_type, enabled, iface): for opt in ['up_cmds', 'pre_up_cmds', 'post_up_cmds', 'down_cmds', 'pre_down_cmds', 'post_down_cmds']: if opt in opts: - iface_data['inet'][opt] = opts[opt] + iface_data[def_addrfam][opt] = opts[opt] for addrfam in ['inet', 'inet6']: if 'addrfam' in iface_data[addrfam] and iface_data[addrfam]['addrfam'] == addrfam: From bb1d2eb85b447386c9c70ad1158cc99eddfb718d Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 18 Oct 2017 14:03:10 -0600 Subject: [PATCH 088/184] Skip tests that are failing on PAM eauth --- tests/unit/test_auth.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/unit/test_auth.py b/tests/unit/test_auth.py index e8d868e7ff..474ecad931 100644 --- a/tests/unit/test_auth.py +++ b/tests/unit/test_auth.py @@ -14,6 +14,7 @@ from tests.support.mock import patch, call, NO_MOCK, NO_MOCK_REASON, MagicMock import salt.master from tests.support.case import ModuleCase from salt import auth +import salt.utils @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -146,6 +147,7 @@ class MasterACLTestCase(ModuleCase): } self.addCleanup(delattr, self, 'valid_clear_load') + @skipIf(salt.utils.is_windows(), 'PAM eauth not available on Windows') def test_master_publish_name(self): ''' Test to ensure a simple name can auth against a given function. @@ -214,6 +216,7 @@ class MasterACLTestCase(ModuleCase): self.clear.publish(self.valid_clear_load) self.assertEqual(self.fire_event_mock.mock_calls, []) + @skipIf(salt.utils.is_windows(), 'PAM eauth not available on Windows') def test_master_minion_glob(self): ''' Test to ensure we can allow access to a given @@ -250,6 +253,7 @@ class MasterACLTestCase(ModuleCase): # Unimplemented pass + @skipIf(salt.utils.is_windows(), 'PAM eauth not available on Windows') def test_args_empty_spec(self): ''' Test simple arg restriction allowed. @@ -267,6 +271,7 @@ class MasterACLTestCase(ModuleCase): self.clear.publish(self.valid_clear_load) self.assertEqual(self.fire_event_mock.call_args[0][0]['fun'], 'test.empty') + @skipIf(salt.utils.is_windows(), 'PAM eauth not available on Windows') def test_args_simple_match(self): ''' Test simple arg restriction allowed. @@ -287,6 +292,7 @@ class MasterACLTestCase(ModuleCase): self.clear.publish(self.valid_clear_load) self.assertEqual(self.fire_event_mock.call_args[0][0]['fun'], 'test.echo') + @skipIf(salt.utils.is_windows(), 'PAM eauth not available on Windows') def test_args_more_args(self): ''' Test simple arg restriction allowed to pass unlisted args. @@ -345,6 +351,7 @@ class MasterACLTestCase(ModuleCase): self.clear.publish(self.valid_clear_load) self.assertEqual(self.fire_event_mock.mock_calls, []) + @skipIf(salt.utils.is_windows(), 'PAM eauth not available on Windows') def test_args_kwargs_match(self): ''' Test simple kwargs restriction allowed. @@ -416,6 +423,7 @@ class MasterACLTestCase(ModuleCase): self.clear.publish(self.valid_clear_load) self.assertEqual(self.fire_event_mock.mock_calls, []) + @skipIf(salt.utils.is_windows(), 'PAM eauth not available on Windows') def test_args_mixed_match(self): ''' Test mixed args and kwargs restriction allowed. @@ -559,6 +567,7 @@ class AuthACLTestCase(ModuleCase): } self.addCleanup(delattr, self, 'valid_clear_load') + @skipIf(salt.utils.is_windows(), 'PAM eauth not available on Windows') def test_acl_simple_allow(self): self.clear.publish(self.valid_clear_load) self.assertEqual(self.auth_check_mock.call_args[0][0], From 1541376c4fda5bd4260aa99130486b0f7df308ab Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Wed, 18 Oct 2017 17:15:42 -0400 Subject: [PATCH 089/184] Add spm build test --- tests/integration/spm/__init__.py | 1 + tests/integration/spm/test_build.py | 65 +++++++++++++++++++++++++++++ tests/support/case.py | 43 +++++++++++++++++++ 3 files changed, 109 insertions(+) create mode 100644 tests/integration/spm/__init__.py create mode 100644 tests/integration/spm/test_build.py diff --git a/tests/integration/spm/__init__.py b/tests/integration/spm/__init__.py new file mode 100644 index 0000000000..40a96afc6f --- /dev/null +++ b/tests/integration/spm/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/tests/integration/spm/test_build.py b/tests/integration/spm/test_build.py new file mode 100644 index 0000000000..79743e0187 --- /dev/null +++ b/tests/integration/spm/test_build.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +''' +Tests for the spm build utility +''' +# Import python libs +from __future__ import absolute_import +import os +import shutil +import textwrap + +# Import Salt Testing libs +from tests.support.case import SPMCase +from tests.support.helpers import destructiveTest + +# Import Salt Libraries +import salt.utils + + +@destructiveTest +class SPMBuildTest(SPMCase): + ''' + Validate the spm build command + ''' + def setUp(self): + self.config = self._spm_config() + self.formula_dir = os.path.join(' '.join(self.config['file_roots']['base']), 'formulas') + self.formula_sls_dir = os.path.join(self.formula_dir, 'apache') + self.formula_sls = os.path.join(self.formula_sls_dir, 'apache.sls') + self.formula_file = os.path.join(self.formula_dir, 'FORMULA') + + dirs = [self.formula_dir, self.formula_sls_dir] + for formula_dir in dirs: + os.makedirs(formula_dir) + + with salt.utils.fopen(self.formula_sls, 'w') as fp: + fp.write(textwrap.dedent('''\ + install-apache: + pkg.installed: + - name: apache2 + ''')) + + with salt.utils.fopen(self.formula_file, 'w') as fp: + fp.write(textwrap.dedent('''\ + name: apache + os: RedHat, Debian, Ubuntu, Suse, FreeBSD + os_family: RedHat, Debian, Suse, FreeBSD + version: 201506 + release: 2 + summary: Formula for installing Apache + description: Formula for installing Apache + ''')) + + def test_spm_build(self): + ''' + test spm build + ''' + build_spm = self.run_spm('build', self.config, self.formula_dir) + spm_file = os.path.join(self.config['spm_build_dir'], 'apache-201506-2.spm') + # Make sure .spm file gets created + self.assertTrue(os.path.exists(spm_file)) + # Make sure formula path dir is created + self.assertTrue(os.path.isdir(self.config['formula_path'])) + + def tearDown(self): + shutil.rmtree(self._tmp_spm) diff --git a/tests/support/case.py b/tests/support/case.py index c35396ee5d..820b2cb557 100644 --- a/tests/support/case.py +++ b/tests/support/case.py @@ -564,6 +564,49 @@ class ShellCase(ShellTestCase, AdaptedConfigurationTestCaseMixin, ScriptPathMixi timeout=timeout) +class SPMCase(TestCase, AdaptedConfigurationTestCaseMixin): + ''' + Class for handling spm commands + ''' + + def _spm_config(self): + self._tmp_spm = tempfile.mkdtemp() + config = self.get_temp_config('minion', **{ + 'spm_logfile': os.path.join(self._tmp_spm, 'log'), + 'spm_repos_config': os.path.join(self._tmp_spm, 'etc', 'spm.repos'), + 'spm_cache_dir': os.path.join(self._tmp_spm, 'cache'), + 'spm_build_dir': os.path.join(self._tmp_spm, 'build'), + 'spm_build_exclude': ['.git'], + 'spm_db_provider': 'sqlite3', + 'spm_files_provider': 'local', + 'spm_db': os.path.join(self._tmp_spm, 'packages.db'), + 'extension_modules': os.path.join(self._tmp_spm, 'modules'), + 'file_roots': {'base': [self._tmp_spm, ]}, + 'formula_path': os.path.join(self._tmp_spm, 'spm'), + 'pillar_path': os.path.join(self._tmp_spm, 'pillar'), + 'reactor_path': os.path.join(self._tmp_spm, 'reactor'), + 'assume_yes': True, + 'force': False, + 'verbose': False, + 'cache': 'localfs', + 'cachedir': os.path.join(self._tmp_spm, 'cache'), + 'spm_repo_dups': 'ignore', + 'spm_share_dir': os.path.join(self._tmp_spm, 'share'), + }) + return config + + def _spm_client(self, config): + import salt.spm + ui = salt.spm.SPMCmdlineInterface() + client = salt.spm.SPMClient(ui, config) + return client + + def run_spm(self, cmd, config, arg=(), minion_tgt='minion'): + client = self._spm_client(config) + spm_cmd = client.run([cmd, arg]) + return spm_cmd + + class ModuleCase(TestCase, SaltClientTestCaseMixin): ''' Execute a module function From cd79e9444e2cd8074746ab7bc4f8854e1f0c46ec Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Wed, 18 Oct 2017 17:21:52 -0400 Subject: [PATCH 090/184] remove unneded kwarg --- tests/support/case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/support/case.py b/tests/support/case.py index 820b2cb557..b620c69c72 100644 --- a/tests/support/case.py +++ b/tests/support/case.py @@ -601,7 +601,7 @@ class SPMCase(TestCase, AdaptedConfigurationTestCaseMixin): client = salt.spm.SPMClient(ui, config) return client - def run_spm(self, cmd, config, arg=(), minion_tgt='minion'): + def run_spm(self, cmd, config, arg=()): client = self._spm_client(config) spm_cmd = client.run([cmd, arg]) return spm_cmd From aed80a3c8df943426ec484ce33e537d3519776e0 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 2 Oct 2017 16:05:05 -0700 Subject: [PATCH 091/184] Provide a grain for total amount of swap for Linux, *BSD, OS X and Solaris/SunOS --- salt/grains/core.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/salt/grains/core.py b/salt/grains/core.py index f54e786fc4..586ee3c3c3 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -414,22 +414,39 @@ def _memdata(osdata): if comps[0].strip() == 'MemTotal': # Use floor division to force output to be an integer grains['mem_total'] = int(comps[1].split()[0]) // 1024 + if comps[0].strip() == 'SwapTotal': + # Use floor division to force output to be an integer + grains['swap_total'] = int(comps[1].split()[0]) // 1024 elif osdata['kernel'] in ('FreeBSD', 'OpenBSD', 'NetBSD', 'Darwin'): sysctl = salt.utils.path.which('sysctl') if sysctl: if osdata['kernel'] == 'Darwin': mem = __salt__['cmd.run']('{0} -n hw.memsize'.format(sysctl)) + swap_total = __salt__['cmd.run']('{0} -n vm.swapusage').split()[2] + if swap_total.endswith('K'): + _power = 2**10 + elif swap_total.endswith('M'): + _power = 2**20 + elif swap_total.endswith('G'): + _power = 2**30 + swap_total = swap_total[:-1] * _power else: mem = __salt__['cmd.run']('{0} -n hw.physmem'.format(sysctl)) + swap_total = __salt__['cmd.run']('{0} -n vm.swap_total') if osdata['kernel'] == 'NetBSD' and mem.startswith('-'): mem = __salt__['cmd.run']('{0} -n hw.physmem64'.format(sysctl)) grains['mem_total'] = int(mem) // 1024 // 1024 + grains['swap_total'] = int(swap_total) // 1024 // 1024 elif osdata['kernel'] == 'SunOS': prtconf = '/usr/sbin/prtconf 2>/dev/null' for line in __salt__['cmd.run'](prtconf, python_shell=True).splitlines(): comps = line.split(' ') if comps[0].strip() == 'Memory' and comps[1].strip() == 'size:': grains['mem_total'] = int(comps[2].strip()) + + swap_cmd = salt.utils.path.which('swap') + swap_total = __salt__['cmd.run']('{0} -s'.format(swap_cmd)).split()[1] + grains['swap_total'] = int(swap_total) // 1024 elif osdata['kernel'] == 'Windows' and HAS_WMI: # get the Total Physical memory as reported by msinfo32 tot_bytes = win32api.GlobalMemoryStatusEx()['TotalPhys'] From bcee88ce777821e0785590c99b13f9f0f92bc525 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 3 Oct 2017 10:09:05 -0700 Subject: [PATCH 092/184] Adding note about new swap_total grain to release notes for Oxyten. --- doc/topics/releases/oxygen.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/topics/releases/oxygen.rst b/doc/topics/releases/oxygen.rst index 4a6cbfb649..7e98290399 100644 --- a/doc/topics/releases/oxygen.rst +++ b/doc/topics/releases/oxygen.rst @@ -55,6 +55,7 @@ The new grains added are: * ``fc_wwn``: Show all fibre channel world wide port names for a host * ``iscsi_iqn``: Show the iSCSI IQN name for a host +* ``swap_total``: Show the configured swap_total for Linux, *BSD, OS X and Solaris/SunOS Grains Changes -------------- From 9f99c304a3b682661ba98886da470db329aeb764 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 13 Oct 2017 08:25:34 -0700 Subject: [PATCH 093/184] moving specific platform memdata code into specific platform functions. --- salt/grains/core.py | 147 ++++++++++++++++++++++++++++++-------------- 1 file changed, 100 insertions(+), 47 deletions(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index 586ee3c3c3..72478bbc54 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -395,63 +395,116 @@ def _sunos_cpudata(): return grains +def _linux_memdata(): + ''' + Return the memory information for Linux-like systems + ''' + grains = {'mem_total': 0, 'swap_total': 0} + + meminfo = '/proc/meminfo' + if os.path.isfile(meminfo): + with salt.utils.files.fopen(meminfo, 'r') as ifile: + for line in ifile: + comps = line.rstrip('\n').split(':') + if not len(comps) > 1: + continue + if comps[0].strip() == 'MemTotal': + # Use floor division to force output to be an integer + grains['mem_total'] = int(comps[1].split()[0]) // 1024 + if comps[0].strip() == 'SwapTotal': + # Use floor division to force output to be an integer + grains['swap_total'] = int(comps[1].split()[0]) // 1024 + return grains + + +def _osx_memdata(): + ''' + Return the memory information for BSD-like systems + ''' + grains = {'mem_total': 0, 'swap_total': 0} + + sysctl = salt.utils.path.which('sysctl') + if sysctl: + mem = __salt__['cmd.run']('{0} -n hw.memsize'.format(sysctl)) + swap_total = __salt__['cmd.run']('{0} -n vm.swapusage').split()[2] + if swap_total.endswith('K'): + _power = 2**10 + elif swap_total.endswith('M'): + _power = 2**20 + elif swap_total.endswith('G'): + _power = 2**30 + swap_total = swap_total[:-1] * _power + + grains['mem_total'] = int(mem) // 1024 // 1024 + grains['swap_total'] = int(swap_total) // 1024 // 1024 + return grains + + +def _bsd_memdata(osdata): + ''' + Return the memory information for BSD-like systems + ''' + grains = {'mem_total': 0, 'swap_total': 0} + + sysctl = salt.utils.path.which('sysctl') + if sysctl: + mem = __salt__['cmd.run']('{0} -n hw.physmem'.format(sysctl)) + swap_total = __salt__['cmd.run']('{0} -n vm.swap_total') + if osdata['kernel'] == 'NetBSD' and mem.startswith('-'): + mem = __salt__['cmd.run']('{0} -n hw.physmem64'.format(sysctl)) + grains['mem_total'] = int(mem) // 1024 // 1024 + grains['swap_total'] = int(swap_total) // 1024 // 1024 + return grains + + +def _sunos_memdata(): + ''' + Return the memory information for SunOS-like systems + ''' + grains = {'mem_total': 0, 'swap_total': 0} + + prtconf = '/usr/sbin/prtconf 2>/dev/null' + for line in __salt__['cmd.run'](prtconf, python_shell=True).splitlines(): + comps = line.split(' ') + if comps[0].strip() == 'Memory' and comps[1].strip() == 'size:': + grains['mem_total'] = int(comps[2].strip()) + + swap_cmd = salt.utils.path.which('swap') + swap_total = __salt__['cmd.run']('{0} -s'.format(swap_cmd)).split()[1] + grains['swap_total'] = int(swap_total) // 1024 + return grains + + +def _windows_memdata(): + ''' + Return the memory information for Windows systems + ''' + grains = {'mem_total': 0} + # get the Total Physical memory as reported by msinfo32 + tot_bytes = win32api.GlobalMemoryStatusEx()['TotalPhys'] + # return memory info in gigabytes + grains['mem_total'] = int(tot_bytes / (1024 ** 2)) + return grains + + def _memdata(osdata): ''' Gather information about the system memory ''' # Provides: # mem_total + # swap_total, for supported systems. grains = {'mem_total': 0} if osdata['kernel'] == 'Linux': - meminfo = '/proc/meminfo' - - if os.path.isfile(meminfo): - with salt.utils.files.fopen(meminfo, 'r') as ifile: - for line in ifile: - comps = line.rstrip('\n').split(':') - if not len(comps) > 1: - continue - if comps[0].strip() == 'MemTotal': - # Use floor division to force output to be an integer - grains['mem_total'] = int(comps[1].split()[0]) // 1024 - if comps[0].strip() == 'SwapTotal': - # Use floor division to force output to be an integer - grains['swap_total'] = int(comps[1].split()[0]) // 1024 - elif osdata['kernel'] in ('FreeBSD', 'OpenBSD', 'NetBSD', 'Darwin'): - sysctl = salt.utils.path.which('sysctl') - if sysctl: - if osdata['kernel'] == 'Darwin': - mem = __salt__['cmd.run']('{0} -n hw.memsize'.format(sysctl)) - swap_total = __salt__['cmd.run']('{0} -n vm.swapusage').split()[2] - if swap_total.endswith('K'): - _power = 2**10 - elif swap_total.endswith('M'): - _power = 2**20 - elif swap_total.endswith('G'): - _power = 2**30 - swap_total = swap_total[:-1] * _power - else: - mem = __salt__['cmd.run']('{0} -n hw.physmem'.format(sysctl)) - swap_total = __salt__['cmd.run']('{0} -n vm.swap_total') - if osdata['kernel'] == 'NetBSD' and mem.startswith('-'): - mem = __salt__['cmd.run']('{0} -n hw.physmem64'.format(sysctl)) - grains['mem_total'] = int(mem) // 1024 // 1024 - grains['swap_total'] = int(swap_total) // 1024 // 1024 + grains.update(_linux_memdata()) + elif osdata['kernel'] in ('FreeBSD', 'OpenBSD', 'NetBSD'): + grains.update(_bsd_memdata(osdata)) + elif osdata['kernel'] == 'Darwin': + grains.update(_osx_memdata()) elif osdata['kernel'] == 'SunOS': - prtconf = '/usr/sbin/prtconf 2>/dev/null' - for line in __salt__['cmd.run'](prtconf, python_shell=True).splitlines(): - comps = line.split(' ') - if comps[0].strip() == 'Memory' and comps[1].strip() == 'size:': - grains['mem_total'] = int(comps[2].strip()) - - swap_cmd = salt.utils.path.which('swap') - swap_total = __salt__['cmd.run']('{0} -s'.format(swap_cmd)).split()[1] - grains['swap_total'] = int(swap_total) // 1024 + grains.update(_sunos_memdata()) elif osdata['kernel'] == 'Windows' and HAS_WMI: - # get the Total Physical memory as reported by msinfo32 - tot_bytes = win32api.GlobalMemoryStatusEx()['TotalPhys'] - # return memory info in gigabytes - grains['mem_total'] = int(tot_bytes / (1024 ** 2)) + grains.update(_windows_memdata()) return grains From 1e312d1b59f92d8493e632b95fd521d5216b62ba Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 18 Oct 2017 12:01:12 -0700 Subject: [PATCH 094/184] Adding tests for _memdata, Linux & *BSD --- salt/grains/core.py | 2 +- tests/unit/grains/test_core.py | 160 +++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 1 deletion(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index 72478bbc54..cae7973a85 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -449,7 +449,7 @@ def _bsd_memdata(osdata): sysctl = salt.utils.path.which('sysctl') if sysctl: mem = __salt__['cmd.run']('{0} -n hw.physmem'.format(sysctl)) - swap_total = __salt__['cmd.run']('{0} -n vm.swap_total') + swap_total = __salt__['cmd.run']('{0} -n vm.swap_total'.format(sysctl)) if osdata['kernel'] == 'NetBSD' and mem.startswith('-'): mem = __salt__['cmd.run']('{0} -n hw.physmem64'.format(sysctl)) grains['mem_total'] = int(mem) // 1024 // 1024 diff --git a/tests/unit/grains/test_core.py b/tests/unit/grains/test_core.py index 3655953e36..5b1e9d8e57 100644 --- a/tests/unit/grains/test_core.py +++ b/tests/unit/grains/test_core.py @@ -463,3 +463,163 @@ PATCHLEVEL = 3 self.assertEqual(os_grains.get('osrelease'), os_release_map['osrelease']) self.assertListEqual(list(os_grains.get('osrelease_info')), os_release_map['osrelease_info']) self.assertEqual(os_grains.get('osmajorrelease'), os_release_map['osmajorrelease']) + + @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux') + def test_linux_memdata(self): + ''' + Test memdata on Linux systems + ''' + _path_exists_map = { + '/proc/1/cmdline': False, + '/proc/meminfo': True + } + _path_isfile_map = { + '/proc/meminfo': True + } + _cmd_run_map = { + 'dpkg --print-architecture': 'amd64' + } + + path_exists_mock = MagicMock(side_effect=lambda x: _path_exists_map[x]) + path_isfile_mock = MagicMock( + side_effect=lambda x: _path_isfile_map.get(x, False) + ) + cmd_run_mock = MagicMock( + side_effect=lambda x: _cmd_run_map[x] + ) + empty_mock = MagicMock(return_value={}) + + _proc_meminfo_file = '''MemTotal: 16277028 kB +SwapTotal: 4789244 kB''' + + orig_import = __import__ + if six.PY2: + built_in = '__builtin__' + else: + built_in = 'builtins' + + def _import_mock(name, *args): + if name == 'lsb_release': + raise ImportError('No module named lsb_release') + return orig_import(name, *args) + + # Skip the first if statement + with patch.object(salt.utils.platform, 'is_proxy', + MagicMock(return_value=False)): + # Skip the selinux/systemd stuff (not pertinent) + with patch.object(core, '_linux_bin_exists', + MagicMock(return_value=False)): + # Skip the init grain compilation (not pertinent) + with patch.object(os.path, 'exists', path_exists_mock): + # Ensure that lsb_release fails to import + with patch('{0}.__import__'.format(built_in), + side_effect=_import_mock): + # Skip all the /etc/*-release stuff (not pertinent) + with patch.object(os.path, 'isfile', path_isfile_mock): + # Make a bunch of functions return empty dicts, + # we don't care about these grains for the + # purposes of this test. + with patch.object( + core, + '_linux_cpudata', + empty_mock): + with patch.object( + core, + '_linux_gpu_data', + empty_mock): + with patch('salt.utils.files.fopen', mock_open()) as _proc_meminfo: + _proc_meminfo.return_value.__iter__.return_value = _proc_meminfo_file.splitlines() + with patch.object( + core, + '_hw_data', + empty_mock): + with patch.object( + core, + '_virtual', + empty_mock): + with patch.object( + core, + '_ps', + empty_mock): + # Mock the osarch + with patch.dict( + core.__salt__, + {'cmd.run': cmd_run_mock}): + os_grains = core.os_data() + + self.assertEqual(os_grains.get('mem_total'), 15895) + self.assertEqual(os_grains.get('swap_total'), 4676) + + def test_bsd_memdata(self): + ''' + Test to memdata on *BSD systems + ''' + _path_exists_map = {} + _path_isfile_map = {} + _cmd_run_map = { + 'freebsd-version -u': '10.3-RELEASE', + '/sbin/sysctl -n hw.physmem': '2121781248', + '/sbin/sysctl -n vm.swap_total': '419430400' + } + + path_exists_mock = MagicMock(side_effect=lambda x: _path_exists_map[x]) + path_isfile_mock = MagicMock( + side_effect=lambda x: _path_isfile_map.get(x, False) + ) + cmd_run_mock = MagicMock( + side_effect=lambda x: _cmd_run_map[x] + ) + empty_mock = MagicMock(return_value={}) + + mock_freebsd_uname = ('FreeBSD', + 'freebsd10.3-hostname-8148', + '10.3-RELEASE', + 'FreeBSD 10.3-RELEASE #0 r297264: Fri Mar 25 02:10:02 UTC 2016 root@releng1.nyi.freebsd.org:/usr/obj/usr/src/sys/GENERIC', + 'amd64', + 'amd64') + + with patch('platform.uname', + MagicMock(return_value=mock_freebsd_uname)): + with patch.object(salt.utils.platform, 'is_linux', + MagicMock(return_value=False)): + with patch.object(salt.utils.platform, 'is_freebsd', + MagicMock(return_value=True)): + # Skip the first if statement + with patch.object(salt.utils.platform, 'is_proxy', + MagicMock(return_value=False)): + # Skip the init grain compilation (not pertinent) + with patch.object(os.path, 'exists', path_exists_mock): + with patch('salt.utils.path.which') as mock: + mock.return_value = '/sbin/sysctl' + # Make a bunch of functions return empty dicts, + # we don't care about these grains for the + # purposes of this test. + with patch.object( + core, + '_bsd_cpudata', + empty_mock): + with patch.object( + core, + '_hw_data', + empty_mock): + with patch.object( + core, + '_zpool_data', + empty_mock): + with patch.object( + core, + '_virtual', + empty_mock): + with patch.object( + core, + '_ps', + empty_mock): + # Mock the osarch + with patch.dict( + core.__salt__, + {'cmd.run': cmd_run_mock}): + os_grains = core.os_data() + + self.assertEqual(os_grains.get('mem_total'), 2023) + self.assertEqual(os_grains.get('swap_total'), 400) + From 8bbb1088e8e5597e22968e5d5f113fb29df90248 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 18 Oct 2017 14:56:28 -0700 Subject: [PATCH 095/184] Fixing lint. --- tests/unit/grains/test_core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/grains/test_core.py b/tests/unit/grains/test_core.py index 5b1e9d8e57..ba9cdb3ba9 100644 --- a/tests/unit/grains/test_core.py +++ b/tests/unit/grains/test_core.py @@ -622,4 +622,3 @@ SwapTotal: 4789244 kB''' self.assertEqual(os_grains.get('mem_total'), 2023) self.assertEqual(os_grains.get('swap_total'), 400) - From 4bfeb7f1d144a8ce283c348459393a26abd4559f Mon Sep 17 00:00:00 2001 From: Benedikt Werner <1benediktwerner@gmail.com> Date: Thu, 19 Oct 2017 03:17:16 +0200 Subject: [PATCH 096/184] Fixed missing import --- tests/unit/modules/test_shadow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/modules/test_shadow.py b/tests/unit/modules/test_shadow.py index 6040b83ab9..62781153c2 100644 --- a/tests/unit/modules/test_shadow.py +++ b/tests/unit/modules/test_shadow.py @@ -10,6 +10,7 @@ from __future__ import absolute_import import salt.utils.platform from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import TestCase, skipIf +from tests.support.helpers import skip_if_not_root # Import salt libs try: From cce7bc9719610d76af4339496bd37b28fa401ceb Mon Sep 17 00:00:00 2001 From: Vernon Cole Date: Wed, 18 Oct 2017 20:06:14 -0600 Subject: [PATCH 097/184] add more unit tests --- tests/unit/cloud/clouds/test_saltify.py | 162 ++++++++++++++++++++++-- 1 file changed, 148 insertions(+), 14 deletions(-) diff --git a/tests/unit/cloud/clouds/test_saltify.py b/tests/unit/cloud/clouds/test_saltify.py index 414a294f3b..3d868a0126 100644 --- a/tests/unit/cloud/clouds/test_saltify.py +++ b/tests/unit/cloud/clouds/test_saltify.py @@ -8,32 +8,53 @@ from __future__ import absolute_import # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin -from tests.support.unit import TestCase -from tests.support.mock import ( - MagicMock, - patch -) +from tests.support.unit import skipIf, TestCase +from tests.support.mock import MagicMock, NO_MOCK, NO_MOCK_REASON, patch, ANY + # Import Salt Libs +import salt.client from salt.cloud.clouds import saltify +TEST_PROFILES = { + 'testprofile1': NotImplemented, + 'testprofile2': {'ssh_username': 'fred', + 'ssh_host': 'betty', + 'remove_config_on_destroy': False, + 'shutdown_on_destroy': True + } + } +TEST_PROFILE_NAMES = ['testprofile1', 'testprofile2'] +@skipIf(NO_MOCK, NO_MOCK_REASON) class SaltifyTestCase(TestCase, LoaderModuleMockMixin): ''' Test cases for salt.cloud.clouds.saltify ''' - # 'create' function tests: 1 + LOCAL_OPTS = { + 'providers': { + 'sfy1': { + 'saltify' : { + 'driver': 'saltify', + 'profiles': TEST_PROFILES + } + }, + }, + 'profiles': TEST_PROFILES, + 'sock_dir': '/var/sockxxx', + 'transport': 'tcp', + } def setup_loader_modules(self): - return { - saltify: { + saltify_globals = { '__active_provider_name__': '', '__utils__': { - 'cloud.bootstrap': MagicMock() - }, - '__opts__': {'providers': {}} - } - } + 'cloud.bootstrap': MagicMock(), + 'cloud.fire_event': MagicMock(), + }, + '__opts__': self.LOCAL_OPTS, + } + return {saltify: saltify_globals} def test_create_no_deploy(self): ''' @@ -43,5 +64,118 @@ class SaltifyTestCase(TestCase, LoaderModuleMockMixin): vm = {'deploy': False, 'driver': 'saltify', 'name': 'dummy' - } + } self.assertTrue(saltify.create(vm)) + + def test_create_and_deploy(self): + ''' + Test if deployment can be done. + ''' + mock_cmd = MagicMock(return_value=True) + with patch.dict( + 'salt.cloud.clouds.saltify.__utils__', + {'cloud.bootstrap': mock_cmd}): + vm_ = {'deploy': True, + 'driver': 'saltify', + 'name': 'testprofile2', + } + result = saltify.create(vm_) + mock_cmd.assert_called_once_with(vm_, ANY) + self.assertTrue(result) + + def test_avail_locations(self): + ''' + Test the avail_locations will always return {} + ''' + self.assertEqual(saltify.avail_locations(), {}) + + + def test_avail_sizes(self): + ''' + Test the avail_sizes will always return {} + ''' + self.assertEqual(saltify.avail_sizes(), {}) + + def test_avail_images(self): + ''' + Test the avail_images will return profiles + ''' + testlist = list(TEST_PROFILE_NAMES) # copy + self.assertEqual( + saltify.avail_images()['Profiles'].sort(), + testlist.sort()) + + + def test_list_nodes(self): + ''' + Test list_nodes will return required fields only + ''' + testgrains = { + 'nodeX1': { + 'id': 'nodeX1', + 'ipv4': [ + '127.0.0.1', '192.1.2.22', '172.16.17.18'], + 'ipv6': [ + '::1', 'fdef:bad:add::f00', '3001:DB8::F00D'], + 'salt-cloud': { + 'driver': 'saltify', + 'provider': 'saltyfy', + 'profile': 'testprofile2' + }, + 'extra_stuff': 'does not belong' + } + } + expected_result = { + 'nodeX1': { + 'id': 'nodeX1', + 'image': 'testprofile2', + 'private_ips': [ + '172.16.17.18', 'fdef:bad:add::f00'], + 'public_ips': [ + '192.1.2.22', '3001:DB8::F00D'], + 'size': '', + 'state': 'running' + } + } + mm_cmd = MagicMock(return_value=testgrains) + lcl = salt.client.LocalClient() + lcl.cmd = mm_cmd + with patch('salt.client.LocalClient', return_value=lcl): + self.assertEqual( + saltify.list_nodes(), + expected_result) + + def test_saltify_reboot(self): + mm_cmd = MagicMock(return_value=True) + lcl = salt.client.LocalClient() + lcl.cmd = mm_cmd + with patch('salt.client.LocalClient', return_value=lcl): + result = saltify.reboot('nodeS1', 'action') + mm_cmd.assert_called_with('nodeS1', 'system.reboot') + self.assertTrue(result) + + def test_saltify_destroy(self): + # destroy calls local.cmd several times and expects + # different results, so we will provide a list of + # results. Each call will get the next value. + # NOTE: this assumes that the call order never changes, + # so to keep things simple, we will not use remove_config... + result_list = [ + {'nodeS1': { # first call returns grains.get + 'driver': 'saltify', + 'provider': 'saltify', + 'profile': 'testprofile2'} + # Note: + # testprofile2 has remove_config_on_destroy: False + # and shutdown_on_destroy: True + }, + {'nodeS1': # last call shuts down the minion + 'system.shutdown worked' }, + ] + mm_cmd = MagicMock(return_value=True, side_effect=result_list) + lcl = salt.client.LocalClient() + lcl.cmd = mm_cmd + with patch('salt.client.LocalClient', return_value=lcl): + result = saltify.destroy('nodeS1', 'action') + mm_cmd.assert_called_with('nodeS1', 'system.shutdown') + self.assertTrue(result) From 2b4f69e1f8fd204d4e0fa99ad7b7b2d9a0933233 Mon Sep 17 00:00:00 2001 From: Vernon Cole Date: Wed, 18 Oct 2017 20:20:36 -0600 Subject: [PATCH 098/184] remove unused parameter --- tests/unit/cloud/clouds/test_saltify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/cloud/clouds/test_saltify.py b/tests/unit/cloud/clouds/test_saltify.py index 3d868a0126..5181f33fb7 100644 --- a/tests/unit/cloud/clouds/test_saltify.py +++ b/tests/unit/cloud/clouds/test_saltify.py @@ -172,7 +172,7 @@ class SaltifyTestCase(TestCase, LoaderModuleMockMixin): {'nodeS1': # last call shuts down the minion 'system.shutdown worked' }, ] - mm_cmd = MagicMock(return_value=True, side_effect=result_list) + mm_cmd = MagicMock(side_effect=result_list) lcl = salt.client.LocalClient() lcl.cmd = mm_cmd with patch('salt.client.LocalClient', return_value=lcl): From 44e37bf6e578cda0930fbcfcde8ae00c3f0b2bdb Mon Sep 17 00:00:00 2001 From: Senthilkumar Eswaran Date: Wed, 18 Oct 2017 22:08:26 -0700 Subject: [PATCH 099/184] Fixing default redis.host in documentation --- salt/modules/redismod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/redismod.py b/salt/modules/redismod.py index a95e1b9f3f..40ebbdc3a1 100644 --- a/salt/modules/redismod.py +++ b/salt/modules/redismod.py @@ -9,7 +9,7 @@ Module to provide redis functionality to Salt .. code-block:: yaml - redis.host: 'localhost' + redis.host: 'salt' redis.port: 6379 redis.db: 0 redis.password: None From ed38db27022d2bf282d3640852cae67bd2c3dcb2 Mon Sep 17 00:00:00 2001 From: Pratik Bandarkar Date: Sat, 7 Oct 2017 18:53:59 +0000 Subject: [PATCH 100/184] create route to send traffic by natting for_GCP A route is a rule that specifies how certain packets should be handled by the virtual network. Routes are associated with virtual machine instances by tag, and the set of routes for a particular VM is called its routing table. For each packet leaving a virtual machine, the system searches that machine's routing table for a single best matching route. This module will create a route to send traffic destined to the Internet through your gateway instance. --- salt/modules/gcp_addon.py | 128 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 salt/modules/gcp_addon.py diff --git a/salt/modules/gcp_addon.py b/salt/modules/gcp_addon.py new file mode 100644 index 0000000000..adba567f65 --- /dev/null +++ b/salt/modules/gcp_addon.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- +''' +A route is a rule that specifies how certain packets should be handled by the +virtual network. Routes are associated with virtual machine instances by tag, +and the set of routes for a particular VM is called its routing table. +For each packet leaving a virtual machine, the system searches that machine's +routing table for a single best matching route. + +This module will create a route to send traffic destined to the Internet +through your gateway instance. + +:codeauthor: :email:`Pratik Bandarkar ` +:maturity: new +:depends: google-api-python-client +:platform: Linux +''' +from __future__ import absolute_import +import logging +log = logging.getLogger(__name__) + +try: + import googleapiclient.discovery + import oauth2client.service_account + HAS_LIB = True +except ImportError: + HAS_LIB = False + +__virtualname__ = 'gcp' + + +def __virtual__(): + ''' + Check for googleapiclient api + ''' + if HAS_LIB is False: + log.info("Required google API's(googleapiclient, oauth2client) not found") + return (HAS_LIB, "Required google API's(googleapiclient, oauth2client) not found") + + +def _get_network(project_id, network_name, service): + ''' + Fetch network selfLink from network name. + ''' + return service.networks().get(project=project_id, + network=network_name).execute() + + +def _get_instance(project_id, instance_zone, name, service): + ''' + Get instance details + ''' + return service.instances().get(project=project_id, + zone=instance_zone, + instance=name).execute() + + +def route_create(credential_file=None, + project_id=None, + name=None, + dest_range=None, + next_hop_instance=None, + instance_zone=None, + tags=None, + network=None, + priority=None + ): + ''' + Create a route to send traffic destined to the Internet through your + gateway instance + + credential_file : string + File location of application default credential. For more information, + refer: https://developers.google.com/identity/protocols/application-default-credentials + project_id : string + Project ID where instance and network resides. + name : string + name of the route to create + next_hop_instance : string + the name of an instance that should handle traffic matching this route. + instance_zone : string + zone where instance("next_hop_instance") resides + network : string + Specifies the network to which the route will be applied. + dest_range : string + The destination range of outgoing packets that the route will apply to. + tags : list + (optional) Identifies the set of instances that this route will apply to. + priority : int + (optional) Specifies the priority of this route relative to other routes. + default=1000 + + CLI Example: + + salt 'salt-master.novalocal' gcp.route_create + credential_file=/root/secret_key.json + project_id=cp100-170315 + name=derby-db-route1 + next_hop_instance=instance-1 + instance_zone=us-central1-a + network=default + dest_range=0.0.0.0/0 + tags=['no-ip'] + priority=700 + + In above example, the instances which are having tag "no-ip" will route the + packet to instance "instance-1"(if packet is intended to other network) + ''' + + credentials = oauth2client.service_account.ServiceAccountCredentials.\ + from_json_keyfile_name(credential_file) + service = googleapiclient.discovery.build('compute', 'v1', + credentials=credentials) + routes = service.routes() + + routes_config = { + 'name': str(name), + 'network': _get_network(project_id, str(network), + service=service)['selfLink'], + 'destRange': str(dest_range), + 'nextHopInstance': _get_instance(project_id, instance_zone, + next_hop_instance, + service=service)['selfLink'], + 'tags': tags, + 'priority': priority + } + route_create_request = routes.insert(project=project_id, + body=routes_config) + return route_create_request.execute() From 6052bef2f47cec56539eff9ac17148a796fd40b6 Mon Sep 17 00:00:00 2001 From: Nicole Thomas Date: Thu, 19 Oct 2017 10:28:27 -0400 Subject: [PATCH 101/184] Fixup warn_until messages This change also updates the `salt.utils.warn_until` path to the new `salt.utils.versions.warn_util` path. --- salt/modules/ssh.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/salt/modules/ssh.py b/salt/modules/ssh.py index 4dbf0f4c2c..80acff2c98 100644 --- a/salt/modules/ssh.py +++ b/salt/modules/ssh.py @@ -25,6 +25,7 @@ import salt.utils.decorators.path import salt.utils.files import salt.utils.path import salt.utils.platform +import salt.utils.versions from salt.exceptions import ( SaltInvocationError, CommandExecutionError, @@ -846,6 +847,8 @@ def get_known_host(user, ''' Return information about known host from the configfile, if any. If there is no such key, return None. + + .. deprecated:: Oxygen CLI Example: @@ -853,10 +856,11 @@ def get_known_host(user, salt '*' ssh.get_known_host ''' - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Neon', - 'get_known_host has been deprecated in favour of ' - 'get_known_host_entries.' + '\'get_known_host\' has been deprecated in favour of ' + '\'get_known_host_entries\'. \'get_known_host\' will be ' + 'removed in Salt Neon.' ) known_hosts = get_known_host_entries(user, hostname, config, port, fingerprint_hash_type) return known_hosts[0] if known_hosts else None @@ -906,6 +910,8 @@ def recv_known_host(hostname, fingerprint_hash_type=None): ''' Retrieve information about host public key from remote server + + .. deprecated:: Oxygen hostname The name of the remote host (e.g. "github.com") @@ -942,10 +948,11 @@ def recv_known_host(hostname, salt '*' ssh.recv_known_host enc= port= ''' - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Neon', - 'recv_known_host has been deprecated in favour of ' - 'recv_known_host_entries.' + '\'recv_known_host\' has been deprecated in favour of ' + '\'recv_known_host_entries\'. \'recv_known_host\' will be ' + 'removed in Salt Neon.' ) known_hosts = recv_known_host_entries(hostname, enc, port, hash_known_hosts, timeout, fingerprint_hash_type) return known_hosts[0] if known_hosts else None From 05d91f44f450679be9cfa69ab323a541d528e3c9 Mon Sep 17 00:00:00 2001 From: Nicole Thomas Date: Thu, 19 Oct 2017 11:44:17 -0400 Subject: [PATCH 102/184] Whitespace fix --- salt/modules/ssh.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/ssh.py b/salt/modules/ssh.py index 80acff2c98..5bcd3d032d 100644 --- a/salt/modules/ssh.py +++ b/salt/modules/ssh.py @@ -847,7 +847,7 @@ def get_known_host(user, ''' Return information about known host from the configfile, if any. If there is no such key, return None. - + .. deprecated:: Oxygen CLI Example: @@ -910,7 +910,7 @@ def recv_known_host(hostname, fingerprint_hash_type=None): ''' Retrieve information about host public key from remote server - + .. deprecated:: Oxygen hostname From 4c8cd9c5070316a79d36f2c6ad8330732f8e2476 Mon Sep 17 00:00:00 2001 From: Vernon Cole Date: Thu, 19 Oct 2017 10:16:28 -0600 Subject: [PATCH 103/184] lint and comment fixes --- tests/unit/cloud/clouds/test_saltify.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/unit/cloud/clouds/test_saltify.py b/tests/unit/cloud/clouds/test_saltify.py index 5181f33fb7..dad6673ded 100644 --- a/tests/unit/cloud/clouds/test_saltify.py +++ b/tests/unit/cloud/clouds/test_saltify.py @@ -18,14 +18,15 @@ from salt.cloud.clouds import saltify TEST_PROFILES = { 'testprofile1': NotImplemented, - 'testprofile2': {'ssh_username': 'fred', - 'ssh_host': 'betty', - 'remove_config_on_destroy': False, - 'shutdown_on_destroy': True + 'testprofile2': { # this profile is used in test_saltify_destroy() + 'ssh_username': 'fred', + 'remove_config_on_destroy': False, # expected for test + 'shutdown_on_destroy': True # expected value for test } } TEST_PROFILE_NAMES = ['testprofile1', 'testprofile2'] + @skipIf(NO_MOCK, NO_MOCK_REASON) class SaltifyTestCase(TestCase, LoaderModuleMockMixin): ''' @@ -34,7 +35,7 @@ class SaltifyTestCase(TestCase, LoaderModuleMockMixin): LOCAL_OPTS = { 'providers': { 'sfy1': { - 'saltify' : { + 'saltify': { 'driver': 'saltify', 'profiles': TEST_PROFILES } @@ -89,7 +90,6 @@ class SaltifyTestCase(TestCase, LoaderModuleMockMixin): ''' self.assertEqual(saltify.avail_locations(), {}) - def test_avail_sizes(self): ''' Test the avail_sizes will always return {} @@ -105,7 +105,6 @@ class SaltifyTestCase(TestCase, LoaderModuleMockMixin): saltify.avail_images()['Profiles'].sort(), testlist.sort()) - def test_list_nodes(self): ''' Test list_nodes will return required fields only @@ -161,16 +160,16 @@ class SaltifyTestCase(TestCase, LoaderModuleMockMixin): # NOTE: this assumes that the call order never changes, # so to keep things simple, we will not use remove_config... result_list = [ - {'nodeS1': { # first call returns grains.get + {'nodeS1': { # first call is grains.get 'driver': 'saltify', 'provider': 'saltify', 'profile': 'testprofile2'} - # Note: - # testprofile2 has remove_config_on_destroy: False - # and shutdown_on_destroy: True }, + # Note: + # testprofile2 has remove_config_on_destroy: False + # and shutdown_on_destroy: True {'nodeS1': # last call shuts down the minion - 'system.shutdown worked' }, + 'a system.shutdown worked message'}, ] mm_cmd = MagicMock(side_effect=result_list) lcl = salt.client.LocalClient() From 56c91f0895e0ec1aea4dee9746e7a4eb6879cb43 Mon Sep 17 00:00:00 2001 From: Sergey Kacheev Date: Fri, 20 Oct 2017 00:11:29 +0700 Subject: [PATCH 104/184] Add requests verify option in vault section This fix allow pass requests 'verify' option from vault configs If vault certificate signed with Intermediate CA, and Intermedia CA sign by internal root CA, requests will fail verifying vault certificate with error: _ssl.c:510: ... routines:SSL3_GET_SERVER_CERTIFICATE:certificate verify failed 'verify' option allow explicitly specify ca-bundle, or disable verifications. http://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification --- salt/modules/vault.py | 5 +++++ salt/runners/vault.py | 4 +++- salt/utils/vault.py | 3 +++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/salt/modules/vault.py b/salt/modules/vault.py index f86925720d..93e1826df8 100644 --- a/salt/modules/vault.py +++ b/salt/modules/vault.py @@ -16,6 +16,7 @@ Functions to interact with Hashicorp Vault. vault: url: https://vault.service.domain:8200 + verify: /etc/ssl/certs/ca-certificates.crt auth: method: token token: 11111111-2222-3333-4444-555555555555 @@ -27,6 +28,10 @@ Functions to interact with Hashicorp Vault. url Url to your Vault installation. Required. + verify + For details please see + http://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification + auth Currently only token auth is supported. The token must be able to create tokens with the policies that should be assigned to minions. Required. diff --git a/salt/runners/vault.py b/salt/runners/vault.py index 0c8d69345a..e35aba2f67 100644 --- a/salt/runners/vault.py +++ b/salt/runners/vault.py @@ -56,8 +56,10 @@ def generate_token(minion_id, signature, impersonated_by_master=False): 'metadata': audit_data } + verify = config.get('verify', None) + log.trace('Sending token creation request to Vault') - response = requests.post(url, headers=headers, json=payload) + response = requests.post(url, headers=headers, json=payload, verify=verify) if response.status_code != 200: return {'error': response.reason} diff --git a/salt/utils/vault.py b/salt/utils/vault.py index 30e66ab20e..1c596dc82c 100644 --- a/salt/utils/vault.py +++ b/salt/utils/vault.py @@ -124,6 +124,9 @@ def make_request(method, resource, profile=None, **args): connection = _get_vault_connection() token, vault_url = connection['token'], connection['url'] + if "verify" not in args: + args["verify"] = __opts__['vault'].get('verify', None) + url = "{0}/{1}".format(vault_url, resource) headers = {'X-Vault-Token': token, 'Content-Type': 'application/json'} response = requests.request(method, url, headers=headers, **args) From 7eef3b35714b77641e98d47666aa0cc3030f0586 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 19 Oct 2017 10:39:11 -0700 Subject: [PATCH 105/184] Adding a copy.deepcopy to the for loop that looks for old jobs to avoid stale jobs ending up in the list. --- salt/utils/schedule.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index 5b182d1c80..ea6628427e 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -1139,7 +1139,8 @@ class Schedule(object): # Sort the list of "whens" from earlier to later schedules _when.sort() - for i in _when: + # Copy the list so we can loop through it + for i in copy.deepcopy(_when): if i < now and len(_when) > 1: # Remove all missed schedules except the latest one. # We need it to detect if it was triggered previously. From 64d2e4f732186049a605b31690e9f5ffa19beb69 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 19 Oct 2017 11:50:23 -0600 Subject: [PATCH 106/184] Fix pickling errors on Windows Moves the 4 sub functions to the global scope --- tests/unit/test_daemons.py | 276 +++++++++++++++++++------------------ 1 file changed, 139 insertions(+), 137 deletions(-) diff --git a/tests/unit/test_daemons.py b/tests/unit/test_daemons.py index a2f3bbc863..cf736acce1 100644 --- a/tests/unit/test_daemons.py +++ b/tests/unit/test_daemons.py @@ -68,6 +68,141 @@ class LoggerMock(object): return False +def _master_exec_test(child_pipe): + def _create_master(): + ''' + Create master instance + :return: + ''' + obj = daemons.Master() + obj.config = {'user': 'dummy', 'hash_type': alg} + for attr in ['start_log_info', 'prepare', 'shutdown', 'master']: + setattr(obj, attr, MagicMock()) + + return obj + + _logger = LoggerMock() + ret = True + with patch('salt.cli.daemons.check_user', MagicMock(return_value=True)): + with patch('salt.cli.daemons.log', _logger): + for alg in ['md5', 'sha1']: + _create_master().start() + ret = ret and _logger.messages \ + and _logger.has_message('Do not use {alg}'.format(alg=alg), + log_type='warning') + + _logger.reset() + + for alg in ['sha224', 'sha256', 'sha384', 'sha512']: + _create_master().start() + ret = ret and _logger.messages \ + and not _logger.has_message('Do not use ') + child_pipe.send(ret) + child_pipe.close() + + +def _minion_exec_test(child_pipe): + def _create_minion(): + ''' + Create minion instance + :return: + ''' + obj = daemons.Minion() + obj.config = {'user': 'dummy', 'hash_type': alg} + for attr in ['start_log_info', 'prepare', 'shutdown']: + setattr(obj, attr, MagicMock()) + setattr(obj, 'minion', MagicMock(restart=False)) + + return obj + + ret = True + _logger = LoggerMock() + with patch('salt.cli.daemons.check_user', MagicMock(return_value=True)): + with patch('salt.cli.daemons.log', _logger): + for alg in ['md5', 'sha1']: + _create_minion().start() + ret = ret and _logger.messages \ + and _logger.has_message('Do not use {alg}'.format(alg=alg), + log_type='warning') + _logger.reset() + + for alg in ['sha224', 'sha256', 'sha384', 'sha512']: + _create_minion().start() + ret = ret and _logger.messages \ + and not _logger.has_message('Do not use ') + + child_pipe.send(ret) + child_pipe.close() + + +def _proxy_exec_test(child_pipe): + def _create_proxy_minion(): + ''' + Create proxy minion instance + :return: + ''' + obj = daemons.ProxyMinion() + obj.config = {'user': 'dummy', 'hash_type': alg} + for attr in ['minion', 'start_log_info', 'prepare', 'shutdown', 'tune_in']: + setattr(obj, attr, MagicMock()) + + obj.minion.restart = False + return obj + + ret = True + _logger = LoggerMock() + with patch('salt.cli.daemons.check_user', MagicMock(return_value=True)): + with patch('salt.cli.daemons.log', _logger): + for alg in ['md5', 'sha1']: + _create_proxy_minion().start() + ret = ret and _logger.messages \ + and _logger.has_message('Do not use {alg}'.format(alg=alg), + log_type='warning') + + _logger.reset() + + for alg in ['sha224', 'sha256', 'sha384', 'sha512']: + _create_proxy_minion().start() + ret = ret and _logger.messages \ + and not _logger.has_message('Do not use ') + child_pipe.send(ret) + child_pipe.close() + + +def _syndic_exec_test(child_pipe): + def _create_syndic(): + ''' + Create syndic instance + :return: + ''' + obj = daemons.Syndic() + obj.config = {'user': 'dummy', 'hash_type': alg} + for attr in ['syndic', 'start_log_info', 'prepare', 'shutdown']: + setattr(obj, attr, MagicMock()) + + return obj + + ret = True + _logger = LoggerMock() + with patch('salt.cli.daemons.check_user', MagicMock(return_value=True)): + with patch('salt.cli.daemons.log', _logger): + for alg in ['md5', 'sha1']: + _create_syndic().start() + ret = ret and _logger.messages \ + and _logger.has_message('Do not use {alg}'.format(alg=alg), + log_type='warning') + + _logger.reset() + + for alg in ['sha224', 'sha256', 'sha384', 'sha512']: + _create_syndic().start() + ret = ret and _logger.messages \ + and not _logger.has_message('Do not use ') + + child_pipe.send(ret) + child_pipe.close() + + @skipIf(NO_MOCK, NO_MOCK_REASON) class DaemonsStarterTestCase(TestCase, SaltClientTestCaseMixin): ''' @@ -87,38 +222,7 @@ class DaemonsStarterTestCase(TestCase, SaltClientTestCaseMixin): :return: ''' - def exec_test(child_pipe): - def _create_master(): - ''' - Create master instance - :return: - ''' - obj = daemons.Master() - obj.config = {'user': 'dummy', 'hash_type': alg} - for attr in ['start_log_info', 'prepare', 'shutdown', 'master']: - setattr(obj, attr, MagicMock()) - - return obj - - _logger = LoggerMock() - ret = True - with patch('salt.cli.daemons.check_user', MagicMock(return_value=True)): - with patch('salt.cli.daemons.log', _logger): - for alg in ['md5', 'sha1']: - _create_master().start() - ret = ret and _logger.messages \ - and _logger.has_message('Do not use {alg}'.format(alg=alg), - log_type='warning') - - _logger.reset() - - for alg in ['sha224', 'sha256', 'sha384', 'sha512']: - _create_master().start() - ret = ret and _logger.messages \ - and not _logger.has_message('Do not use ') - child_pipe.send(ret) - child_pipe.close() - self._multiproc_exec_test(exec_test) + self._multiproc_exec_test(_master_exec_test) def test_minion_daemon_hash_type_verified(self): ''' @@ -126,41 +230,7 @@ class DaemonsStarterTestCase(TestCase, SaltClientTestCaseMixin): :return: ''' - - def exec_test(child_pipe): - def _create_minion(): - ''' - Create minion instance - :return: - ''' - obj = daemons.Minion() - obj.config = {'user': 'dummy', 'hash_type': alg} - for attr in ['start_log_info', 'prepare', 'shutdown']: - setattr(obj, attr, MagicMock()) - setattr(obj, 'minion', MagicMock(restart=False)) - - return obj - - ret = True - _logger = LoggerMock() - with patch('salt.cli.daemons.check_user', MagicMock(return_value=True)): - with patch('salt.cli.daemons.log', _logger): - for alg in ['md5', 'sha1']: - _create_minion().start() - ret = ret and _logger.messages \ - and _logger.has_message('Do not use {alg}'.format(alg=alg), - log_type='warning') - _logger.reset() - - for alg in ['sha224', 'sha256', 'sha384', 'sha512']: - _create_minion().start() - ret = ret and _logger.messages \ - and not _logger.has_message('Do not use ') - - child_pipe.send(ret) - child_pipe.close() - - self._multiproc_exec_test(exec_test) + self._multiproc_exec_test(_minion_exec_test) def test_proxy_minion_daemon_hash_type_verified(self): ''' @@ -168,41 +238,7 @@ class DaemonsStarterTestCase(TestCase, SaltClientTestCaseMixin): :return: ''' - - def exec_test(child_pipe): - def _create_proxy_minion(): - ''' - Create proxy minion instance - :return: - ''' - obj = daemons.ProxyMinion() - obj.config = {'user': 'dummy', 'hash_type': alg} - for attr in ['minion', 'start_log_info', 'prepare', 'shutdown', 'tune_in']: - setattr(obj, attr, MagicMock()) - - obj.minion.restart = False - return obj - - ret = True - _logger = LoggerMock() - with patch('salt.cli.daemons.check_user', MagicMock(return_value=True)): - with patch('salt.cli.daemons.log', _logger): - for alg in ['md5', 'sha1']: - _create_proxy_minion().start() - ret = ret and _logger.messages \ - and _logger.has_message('Do not use {alg}'.format(alg=alg), - log_type='warning') - - _logger.reset() - - for alg in ['sha224', 'sha256', 'sha384', 'sha512']: - _create_proxy_minion().start() - ret = ret and _logger.messages \ - and not _logger.has_message('Do not use ') - child_pipe.send(ret) - child_pipe.close() - - self._multiproc_exec_test(exec_test) + self._multiproc_exec_test(_proxy_exec_test) def test_syndic_daemon_hash_type_verified(self): ''' @@ -210,38 +246,4 @@ class DaemonsStarterTestCase(TestCase, SaltClientTestCaseMixin): :return: ''' - - def exec_test(child_pipe): - def _create_syndic(): - ''' - Create syndic instance - :return: - ''' - obj = daemons.Syndic() - obj.config = {'user': 'dummy', 'hash_type': alg} - for attr in ['syndic', 'start_log_info', 'prepare', 'shutdown']: - setattr(obj, attr, MagicMock()) - - return obj - - ret = True - _logger = LoggerMock() - with patch('salt.cli.daemons.check_user', MagicMock(return_value=True)): - with patch('salt.cli.daemons.log', _logger): - for alg in ['md5', 'sha1']: - _create_syndic().start() - ret = ret and _logger.messages \ - and _logger.has_message('Do not use {alg}'.format(alg=alg), - log_type='warning') - - _logger.reset() - - for alg in ['sha224', 'sha256', 'sha384', 'sha512']: - _create_syndic().start() - ret = ret and _logger.messages \ - and not _logger.has_message('Do not use ') - - child_pipe.send(ret) - child_pipe.close() - - self._multiproc_exec_test(exec_test) + self._multiproc_exec_test(_syndic_exec_test) From 5c66a38b501e2f56854e5c7743d408a84b5d98a1 Mon Sep 17 00:00:00 2001 From: Tom Williams Date: Thu, 19 Oct 2017 13:14:51 -0400 Subject: [PATCH 107/184] INFRA-5825 - add delete_{ingress,egress}_rules params to boto_secgroup --- salt/states/boto_secgroup.py | 65 +++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/salt/states/boto_secgroup.py b/salt/states/boto_secgroup.py index aa2acdf52b..8ab4abc390 100644 --- a/salt/states/boto_secgroup.py +++ b/salt/states/boto_secgroup.py @@ -126,6 +126,8 @@ def present( vpc_name=None, rules=None, rules_egress=None, + delete_ingress_rules=True, + delete_egress_rules=True, region=None, key=None, keyid=None, @@ -160,6 +162,16 @@ def present( the egress rules will be unmanaged. If set to an empty list, ``[]``, then all egress rules will be removed. + delete_ingress_rules + Some tools (EMR comes to mind) insist on adding rules on-the-fly, which + salt will happily remove on the next run. Set this param to False to + avoid deleting rules which were added outside of salt. + + delete_egress_rules + Some tools (EMR comes to mind) insist on adding rules on-the-fly, which + salt will happily remove on the next run. Set this param to False to + avoid deleting rules which were added outside of salt. + region Region to connect to. @@ -191,17 +203,18 @@ def present( elif ret['result'] is None: return ret if rules is not None: - _ret = _rules_present(name, rules, vpc_id=vpc_id, vpc_name=vpc_name, - region=region, key=key, keyid=keyid, - profile=profile) + _ret = _rules_present(name, rules, delete_ingress_rules, vpc_id=vpc_id, + vpc_name=vpc_name, region=region, key=key, + keyid=keyid, profile=profile) ret['changes'] = dictupdate.update(ret['changes'], _ret['changes']) ret['comment'] = ' '.join([ret['comment'], _ret['comment']]) if not _ret['result']: ret['result'] = _ret['result'] if rules_egress is not None: - _ret = _rules_egress_present(name, rules_egress, vpc_id=vpc_id, - vpc_name=vpc_name, region=region, key=key, - keyid=keyid, profile=profile) + _ret = _rules_egress_present(name, rules_egress, delete_egress_rules, + vpc_id=vpc_id, vpc_name=vpc_name, + region=region, key=key, keyid=keyid, + profile=profile) ret['changes'] = dictupdate.update(ret['changes'], _ret['changes']) ret['comment'] = ' '.join([ret['comment'], _ret['comment']]) if not _ret['result']: @@ -389,13 +402,14 @@ def _get_rule_changes(rules, _rules): return (to_delete, to_create) -def _rules_present(name, rules, vpc_id=None, vpc_name=None, - region=None, key=None, keyid=None, profile=None): +def _rules_present(name, rules, delete_ingress_rules=True, vpc_id=None, + vpc_name=None, region=None, key=None, keyid=None, profile=None): ''' given a group name or group name and vpc_id (or vpc name): 1. get lists of desired rule changes (using _get_rule_changes) - 2. delete/revoke or authorize/create rules - 3. return 'old' and 'new' group rules + 2. authorize/create rules missing rules + 3. if delete_ingress_rules is True, delete/revoke non-requested rules + 4. return 'old' and 'new' group rules ''' ret = {'result': True, 'comment': '', 'changes': {}} sg = __salt__['boto_secgroup.get_config'](name=name, group_id=None, region=region, key=key, @@ -424,11 +438,13 @@ def _rules_present(name, rules, vpc_id=None, vpc_name=None, # rules = rules that exist in salt state # sg['rules'] = that exist in present group to_delete, to_create = _get_rule_changes(rules, sg['rules']) + to_delete = to_delete if delete_ingress_rules else [] if to_create or to_delete: if __opts__['test']: msg = """Security group {0} set to have rules modified. To be created: {1} - To be deleted: {2}""".format(name, pprint.pformat(to_create), pprint.pformat(to_delete)) + To be deleted: {2}""".format(name, pprint.pformat(to_create), + pprint.pformat(to_delete)) ret['comment'] = msg ret['result'] = None return ret @@ -470,13 +486,14 @@ def _rules_present(name, rules, vpc_id=None, vpc_name=None, return ret -def _rules_egress_present(name, rules_egress, vpc_id=None, vpc_name=None, - region=None, key=None, keyid=None, profile=None): +def _rules_egress_present(name, rules_egress, delete_egress_rules=True, vpc_id=None, + vpc_name=None, region=None, key=None, keyid=None, profile=None): ''' given a group name or group name and vpc_id (or vpc name): 1. get lists of desired rule changes (using _get_rule_changes) - 2. delete/revoke or authorize/create rules - 3. return 'old' and 'new' group rules + 2. authorize/create missing rules + 3. if delete_egress_rules is True, delete/revoke non-requested rules + 4. return 'old' and 'new' group rules ''' ret = {'result': True, 'comment': '', 'changes': {}} sg = __salt__['boto_secgroup.get_config'](name=name, group_id=None, region=region, key=key, @@ -504,20 +521,20 @@ def _rules_egress_present(name, rules_egress, vpc_id=None, vpc_name=None, rule['source_group_group_id'] = _group_id # rules_egress = rules that exist in salt state # sg['rules_egress'] = that exist in present group - to_delete_egress, to_create_egress = _get_rule_changes( - rules_egress, sg['rules_egress'] - ) - if to_create_egress or to_delete_egress: + to_delete, to_create = _get_rule_changes(rules_egress, sg['rules_egress']) + to_delete = to_delete if delete_egress_rules else [] + if to_create or to_delete: if __opts__['test']: msg = """Security group {0} set to have rules modified. To be created: {1} - To be deleted: {2}""".format(name, pprint.pformat(to_create_egress), pprint.pformat(to_delete_egress)) + To be deleted: {2}""".format(name, pprint.pformat(to_create), + pprint.pformat(to_delete)) ret['comment'] = msg ret['result'] = None return ret - if to_delete_egress: + if to_delete: deleted = True - for rule in to_delete_egress: + for rule in to_delete: _deleted = __salt__['boto_secgroup.revoke']( name, vpc_id=vpc_id, vpc_name=vpc_name, region=region, key=key, keyid=keyid, profile=profile, egress=True, **rule) @@ -530,9 +547,9 @@ def _rules_egress_present(name, rules_egress, vpc_id=None, vpc_name=None, msg = 'Failed to remove egress rule on {0} security group.' ret['comment'] = ' '.join([ret['comment'], msg.format(name)]) ret['result'] = False - if to_create_egress: + if to_create: created = True - for rule in to_create_egress: + for rule in to_create: _created = __salt__['boto_secgroup.authorize']( name, vpc_id=vpc_id, vpc_name=vpc_name, region=region, key=key, keyid=keyid, profile=profile, egress=True, **rule) From 0692f442dbaf5a0b16ce696c8947d73ed14965a9 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 19 Oct 2017 13:54:03 -0500 Subject: [PATCH 108/184] yumpkg: Check pkgname instead of name to see if it is a kernel pkg We should have been checking pkgname instead of name all along, but this never bit us until 6638749 started passing `None` for the name param. --- salt/modules/yumpkg.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 9b2c541953..0a7b3575c6 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -1230,24 +1230,26 @@ def install(name=None, to_install.append((pkgname, pkgstr)) break else: - if re.match('kernel(-.+)?', name): - # kernel and its subpackages support multiple - # installs as their paths do not conflict. - # Performing a yum/dnf downgrade will be a no-op - # so just do an install instead. It will fail if - # there are other interdependencies that have - # conflicts, and that's OK. We don't want to force - # anything, we just want to properly handle it if - # someone tries to install a kernel/kernel-devel of - # a lower version than the currently-installed one. - # TODO: find a better way to determine if a package - # supports multiple installs. - to_install.append((pkgname, pkgstr)) - else: - # None of the currently-installed versions are - # greater than the specified version, so this is a - # downgrade. - to_downgrade.append((pkgname, pkgstr)) + if pkgname is not None: + if re.match('kernel(-.+)?', pkgname): + # kernel and its subpackages support multiple + # installs as their paths do not conflict. + # Performing a yum/dnf downgrade will be a + # no-op so just do an install instead. It will + # fail if there are other interdependencies + # that have conflicts, and that's OK. We don't + # want to force anything, we just want to + # properly handle it if someone tries to + # install a kernel/kernel-devel of a lower + # version than the currently-installed one. + # TODO: find a better way to determine if a + # package supports multiple installs. + to_install.append((pkgname, pkgstr)) + else: + # None of the currently-installed versions are + # greater than the specified version, so this + # is a downgrade. + to_downgrade.append((pkgname, pkgstr)) def _add_common_args(cmd): ''' From d78f27466d998253fef70c63cfb08edcc9c10c91 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 29 Sep 2017 14:03:42 -0600 Subject: [PATCH 109/184] Fix `unit.states.test_mount` for Windows Use `os.path.realpath` to convert paths Mock `os.path.exists` --- tests/unit/states/test_mount.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/unit/states/test_mount.py b/tests/unit/states/test_mount.py index 65173ea698..246a6131db 100644 --- a/tests/unit/states/test_mount.py +++ b/tests/unit/states/test_mount.py @@ -17,6 +17,7 @@ from tests.support.mock import ( # Import Salt Libs import salt.states.mount as mount +import salt.utils @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -33,11 +34,11 @@ class MountTestCase(TestCase, LoaderModuleMockMixin): ''' Test to verify that a device is mounted. ''' - name = '/mnt/sdb' - device = '/dev/sdb5' + name = os.path.realpath('/mnt/sdb') + device = os.path.realpath('/dev/sdb5') fstype = 'xfs' - name2 = '/mnt/cifs' + name2 = os.path.realpath('/mnt/cifs') device2 = '//SERVER/SHARE/' fstype2 = 'cifs' opts2 = ['noowners'] @@ -62,12 +63,11 @@ class MountTestCase(TestCase, LoaderModuleMockMixin): mock_str = MagicMock(return_value='salt') mock_user = MagicMock(return_value={'uid': 510}) mock_group = MagicMock(return_value={'gid': 100}) - umount1 = ("Forced unmount because devices don't match. " - "Wanted: /dev/sdb6, current: /dev/sdb5, /dev/sdb5") with patch.dict(mount.__grains__, {'os': 'Darwin'}): with patch.dict(mount.__salt__, {'mount.active': mock_mnt, 'cmd.run_all': mock_ret, - 'mount.umount': mock_f}): + 'mount.umount': mock_f}), \ + patch('os.path.exists', MagicMock(return_value=True)): comt = ('Unable to find device with label /dev/sdb5.') ret.update({'comment': comt}) self.assertDictEqual(mount.mounted(name, 'LABEL=/dev/sdb5', @@ -81,7 +81,7 @@ class MountTestCase(TestCase, LoaderModuleMockMixin): ret) with patch.dict(mount.__opts__, {'test': False}): - comt = ('Unable to unmount /mnt/sdb: False.') + comt = ('Unable to unmount {0}: False.'.format(name)) umount = ('Forced unmount and mount because' ' options (noowners) changed') ret.update({'comment': comt, 'result': False, @@ -89,16 +89,19 @@ class MountTestCase(TestCase, LoaderModuleMockMixin): self.assertDictEqual(mount.mounted(name, device, 'nfs'), ret) + umount1 = ("Forced unmount because devices don't match. " + "Wanted: {0}, current: {1}, {1}".format(os.path.realpath('/dev/sdb6'), device)) comt = ('Unable to unmount') ret.update({'comment': comt, 'result': None, 'changes': {'umount': umount1}}) - self.assertDictEqual(mount.mounted(name, '/dev/sdb6', + self.assertDictEqual(mount.mounted(name, os.path.realpath('/dev/sdb6'), fstype, opts=[]), ret) with patch.dict(mount.__salt__, {'mount.active': mock_emt, 'mount.mount': mock_str, 'mount.set_automaster': mock}): - with patch.dict(mount.__opts__, {'test': True}): + with patch.dict(mount.__opts__, {'test': True}), \ + patch('os.path.exists', MagicMock(return_value=False)): comt = ('{0} does not exist and would not be created'.format(name)) ret.update({'comment': comt, 'changes': {}}) self.assertDictEqual(mount.mounted(name, device, @@ -117,14 +120,16 @@ class MountTestCase(TestCase, LoaderModuleMockMixin): self.assertDictEqual(mount.mounted(name, device, fstype), ret) - with patch.dict(mount.__opts__, {'test': True}): + with patch.dict(mount.__opts__, {'test': True}), \ + patch('os.path.exists', MagicMock(return_value=False)): comt = ('{0} does not exist and would neither be created nor mounted. ' '{0} needs to be written to the fstab in order to be made persistent.'.format(name)) ret.update({'comment': comt, 'result': None}) self.assertDictEqual(mount.mounted(name, device, fstype, mount=False), ret) - with patch.dict(mount.__opts__, {'test': False}): + with patch.dict(mount.__opts__, {'test': False}), \ + patch('os.path.exists', MagicMock(return_value=False)): comt = ('{0} not present and not mounted. ' 'Entry already exists in the fstab.'.format(name)) ret.update({'comment': comt, 'result': True}) From a862e0bf2dbc586013c793c473791ffcb3b499d9 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 29 Sep 2017 14:06:48 -0600 Subject: [PATCH 110/184] Remove unneeded import --- tests/unit/states/test_mount.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/states/test_mount.py b/tests/unit/states/test_mount.py index 246a6131db..d094747cd8 100644 --- a/tests/unit/states/test_mount.py +++ b/tests/unit/states/test_mount.py @@ -17,7 +17,6 @@ from tests.support.mock import ( # Import Salt Libs import salt.states.mount as mount -import salt.utils @skipIf(NO_MOCK, NO_MOCK_REASON) From f9ad446019186b80643f09e694b5198d846e12fd Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 3 Oct 2017 14:25:09 -0600 Subject: [PATCH 111/184] Fix win_lgpo execution module The _processPolicyDefinitions fuction loads the adml files for the corresponding admx. The adml file usually resides in a sub folder in the C:\Windows\PolicyDefinitions directory that is the language code. The default is `en-US`. When the System Center Operations Manager (SCOM) agent is installed, it adds some addition policy definitions. However, the corresponding adml files are not in the standard location (`en-US`). Instead they are placed in the `en` directory. This PR will cause the win_lgpo module to check in the `en-US` directory first and then check the `en` directory if still not found. --- salt/modules/win_lgpo.py | 50 +++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 2b834227e9..68c1f17106 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -2773,22 +2773,44 @@ def _processPolicyDefinitions(policy_def_path='c:\\Windows\\PolicyDefinitions', temp_ns = policy_ns temp_ns = _updateNamespace(temp_ns, this_namespace) policydefs_policyns_xpath(t_policy_definitions)[0].append(temp_ns) - adml_file = os.path.join(root, display_language, os.path.splitext(t_admfile)[0] + '.adml') + + # We need to make sure the adml file exists. First we'll check + # the passed display_language (eg: en-US). Then we'll try the + # abbreviated version (en) to account for alternate locations. + # We'll do the same for the display_language_fallback (en_US). + adml_file = os.path.join(root, display_language, + os.path.splitext(t_admfile)[0] + '.adml') if not __salt__['file.file_exists'](adml_file): msg = ('An ADML file in the specified ADML language "{0}" ' - 'does not exist for the ADMX "{1}", the fallback ' - 'language will be tried.') + 'does not exist for the ADMX "{1}", the abbreviated ' + 'language code will be tried.') log.info(msg.format(display_language, t_admfile)) - adml_file = os.path.join(root, - display_language_fallback, - os.path.splitext(t_admfile)[0] + '.adml') + + adml_file = os.path.join(root, display_language[:2], + os.path.splitext(t_admfile)[0] + '.adml') if not __salt__['file.file_exists'](adml_file): - msg = ('An ADML file in the specified ADML language ' - '"{0}" and the fallback language "{1}" do not ' - 'exist for the ADMX "{2}".') - raise SaltInvocationError(msg.format(display_language, - display_language_fallback, - t_admfile)) + msg = ('An ADML file in the specified ADML language code "{0}" ' + 'does not exist for the ADMX "{1}", the fallback ' + 'language will be tried.') + log.info(msg.format(display_language[:2], t_admfile)) + + adml_file = os.path.join(root, display_language_fallback, + os.path.splitext(t_admfile)[0] + '.adml') + if not __salt__['file.file_exists'](adml_file): + msg = ('An ADML file in the specified ADML fallback language "{0}" ' + 'does not exist for the ADMX "{1}", the abbreviated' + 'fallback language code will be tried.') + log.info(msg.format(display_language_fallback, t_admfile)) + + adml_file = os.path.join(root, display_language_fallback[:2], + os.path.splitext(t_admfile)[0] + '.adml') + if not __salt__['file.file_exists'](adml_file): + msg = ('An ADML file in the specified ADML language ' + '"{0}" and the fallback language "{1}" do not ' + 'exist for the ADMX "{2}".') + raise SaltInvocationError(msg.format(display_language, + display_language_fallback, + t_admfile)) try: xmltree = lxml.etree.parse(adml_file) except lxml.etree.XMLSyntaxError: @@ -2796,8 +2818,8 @@ def _processPolicyDefinitions(policy_def_path='c:\\Windows\\PolicyDefinitions', try: xmltree = _remove_unicode_encoding(adml_file) except Exception: - msg = ('An error was found while processing adml file {0}, all policy' - ' languange data from this file will be unavailable via this module') + msg = ('An error was found while processing adml file {0}, all policy ' + 'language data from this file will be unavailable via this module') log.error(msg.format(adml_file)) continue if None in namespaces: From 5ec58c620029ff2b231128e2bbbf54d63ccd6eb7 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 5 Oct 2017 15:14:00 -0600 Subject: [PATCH 112/184] Use System Install Language as default fallback --- salt/modules/win_lgpo.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 68c1f17106..b2cd813668 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -44,6 +44,8 @@ from __future__ import absolute_import import os import logging import re +import locale +import ctypes # Import salt libs import salt.utils @@ -117,6 +119,15 @@ try: ADMX_DISPLAYNAME_SEARCH_XPATH = etree.XPath('//*[local-name() = "policy" and @*[local-name() = "displayName"] = $display_name and (@*[local-name() = "class"] = "Both" or @*[local-name() = "class"] = $registry_class) ]') PRESENTATION_ANCESTOR_XPATH = etree.XPath('ancestor::*[local-name() = "presentation"]') TEXT_ELEMENT_XPATH = etree.XPath('.//*[local-name() = "text"]') + # Get the System Install Language + # https://msdn.microsoft.com/en-us/library/dd318123(VS.85).aspx + # local.windows_locale is a dict + # GetSystemDefaultUILanguage() returns a 4 digit language code that + # corresponds to an entry in the dict + # Not available in win32api, so we have to use ctypes + # Default to `en-US` (1033) + windll = ctypes.windll.kernel32 + INSTALL_LANGUAGE = locale.windows_locale.get(windll.GetSystemDefaultUILanguage(), 1033) except ImportError: HAS_WINDOWS_MODULES = False @@ -2709,7 +2720,8 @@ def _processPolicyDefinitions(policy_def_path='c:\\Windows\\PolicyDefinitions', helper function to process all ADMX files in the specified policy_def_path and build a single XML doc that we can search/use for ADMX policy processing ''' - display_language_fallback = 'en-US' + # Fallback to the System Install Language + display_language_fallback = INSTALL_LANGUAGE t_policy_definitions = lxml.etree.Element('policyDefinitions') t_policy_definitions.append(lxml.etree.Element('categories')) t_policy_definitions.append(lxml.etree.Element('policies')) From 5471bd521fe72868790cd67c226949d06c8e56d7 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 6 Oct 2017 13:41:32 -0600 Subject: [PATCH 113/184] Fix problem with file handle Use with io.open so the handle is realeased automatically.... Use `closed` to detect failure to release the file handle and continue to close until successful Use `__opts__` to get `cachedir` instead of __salt__ --- salt/modules/win_lgpo.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index b2cd813668..66766dd00d 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -41,6 +41,7 @@ Current known limitations # Import python libs from __future__ import absolute_import +import io import os import logging import re @@ -2862,13 +2863,14 @@ def _findOptionValueInSeceditFile(option): ''' try: _d = uuid.uuid4().hex - _tfile = '{0}\\{1}'.format(__salt__['config.get']('cachedir'), + _tfile = '{0}\\{1}'.format(__opts__['cachedir'], 'salt-secedit-dump-{0}.txt'.format(_d)) _ret = __salt__['cmd.run']('secedit /export /cfg {0}'.format(_tfile)) if _ret: - _reader = codecs.open(_tfile, 'r', encoding='utf-16') - _secdata = _reader.readlines() - _reader.close() + with io.open(_tfile, encoding='utf-16') as _reader: + _secdata = _reader.readlines() + while not _reader.closed: + _reader.close() if __salt__['file.file_exists'](_tfile): _ret = __salt__['file.remove'](_tfile) for _line in _secdata: @@ -2886,9 +2888,9 @@ def _importSeceditConfig(infdata): ''' try: _d = uuid.uuid4().hex - _tSdbfile = '{0}\\{1}'.format(__salt__['config.get']('cachedir'), + _tSdbfile = '{0}\\{1}'.format(__opts__['cachedir'], 'salt-secedit-import-{0}.sdb'.format(_d)) - _tInfFile = '{0}\\{1}'.format(__salt__['config.get']('cachedir'), + _tInfFile = '{0}\\{1}'.format(__opts__['cachedir'], 'salt-secedit-config-{0}.inf'.format(_d)) # make sure our temp files don't already exist if __salt__['file.file_exists'](_tSdbfile): From b96186d60dc471586bacfe9031563ae8a71a7790 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 9 Oct 2017 13:19:04 -0600 Subject: [PATCH 114/184] Fix INSTALL_LANGUAGE Replace `_` with `-` Instead of taking the first 2 elements of the language code to find the abbreviated language code, do a split off the `-` and take the first element. --- salt/modules/win_lgpo.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 66766dd00d..49b3a3e216 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -128,7 +128,8 @@ try: # Not available in win32api, so we have to use ctypes # Default to `en-US` (1033) windll = ctypes.windll.kernel32 - INSTALL_LANGUAGE = locale.windows_locale.get(windll.GetSystemDefaultUILanguage(), 1033) + INSTALL_LANGUAGE = locale.windows_locale.get( + windll.GetSystemDefaultUILanguage(), 1033).replace('_', '-') except ImportError: HAS_WINDOWS_MODULES = False @@ -2799,7 +2800,7 @@ def _processPolicyDefinitions(policy_def_path='c:\\Windows\\PolicyDefinitions', 'language code will be tried.') log.info(msg.format(display_language, t_admfile)) - adml_file = os.path.join(root, display_language[:2], + adml_file = os.path.join(root, display_language.split('-')[0], os.path.splitext(t_admfile)[0] + '.adml') if not __salt__['file.file_exists'](adml_file): msg = ('An ADML file in the specified ADML language code "{0}" ' @@ -2815,7 +2816,7 @@ def _processPolicyDefinitions(policy_def_path='c:\\Windows\\PolicyDefinitions', 'fallback language code will be tried.') log.info(msg.format(display_language_fallback, t_admfile)) - adml_file = os.path.join(root, display_language_fallback[:2], + adml_file = os.path.join(root, display_language_fallback.split('-')[0], os.path.splitext(t_admfile)[0] + '.adml') if not __salt__['file.file_exists'](adml_file): msg = ('An ADML file in the specified ADML language ' From d5bec9912693cde42813f157427180a12334848d Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 9 Oct 2017 14:53:56 -0600 Subject: [PATCH 115/184] Fix some lint --- salt/modules/win_lgpo.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 49b3a3e216..9548d02062 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -34,7 +34,6 @@ Current known limitations - pywin32 Python module - lxml - uuid - - codecs - struct - salt.modules.reg ''' @@ -93,7 +92,6 @@ try: import win32net import win32security import uuid - import codecs import lxml import struct from lxml import etree @@ -2872,6 +2870,7 @@ def _findOptionValueInSeceditFile(option): _secdata = _reader.readlines() while not _reader.closed: _reader.close() + _reader. if __salt__['file.file_exists'](_tfile): _ret = __salt__['file.remove'](_tfile) for _line in _secdata: From 020c2a2b8525bd830e229922a096cb6d5e6c42ad Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 9 Oct 2017 14:56:56 -0600 Subject: [PATCH 116/184] Fix syntax error --- salt/modules/win_lgpo.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 9548d02062..6f18e5d259 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -2870,7 +2870,6 @@ def _findOptionValueInSeceditFile(option): _secdata = _reader.readlines() while not _reader.closed: _reader.close() - _reader. if __salt__['file.file_exists'](_tfile): _ret = __salt__['file.remove'](_tfile) for _line in _secdata: From 261dba347d849eb813840b9d9b9c03d490cb4c49 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 11 Oct 2017 11:17:50 -0600 Subject: [PATCH 117/184] Put the file.remove in a try/except/else block --- salt/modules/win_lgpo.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 6f18e5d259..b89c660f1d 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -46,6 +46,7 @@ import logging import re import locale import ctypes +import time # Import salt libs import salt.utils @@ -2868,10 +2869,17 @@ def _findOptionValueInSeceditFile(option): if _ret: with io.open(_tfile, encoding='utf-16') as _reader: _secdata = _reader.readlines() - while not _reader.closed: - _reader.close() if __salt__['file.file_exists'](_tfile): - _ret = __salt__['file.remove'](_tfile) + for _ in range(5): + try: + __salt__['file.remove'](_tfile) + except WindowsError: + time.sleep(.1) + continue + else: + break + else: + log.error('error occured removing {0}'.format(_tfile)) for _line in _secdata: if _line.startswith(option): return True, _line.split('=')[1].strip() From 91258cd6a885c69dfd5195281e4e2489af7f46a9 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 12 Oct 2017 13:14:57 -0600 Subject: [PATCH 118/184] Fix typo --- salt/modules/win_lgpo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index b89c660f1d..ffc73f2f01 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -2879,7 +2879,7 @@ def _findOptionValueInSeceditFile(option): else: break else: - log.error('error occured removing {0}'.format(_tfile)) + log.error('error occurred removing {0}'.format(_tfile)) for _line in _secdata: if _line.startswith(option): return True, _line.split('=')[1].strip() From 0040082d0aa4f7b9811153e56c436db265ac288c Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 12 Oct 2017 13:17:39 -0600 Subject: [PATCH 119/184] Fix pylint error --- salt/modules/win_lgpo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index ffc73f2f01..7d2998c272 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -2873,7 +2873,7 @@ def _findOptionValueInSeceditFile(option): for _ in range(5): try: __salt__['file.remove'](_tfile) - except WindowsError: + except WindowsError: # pylint: disable=E0602 time.sleep(.1) continue else: From 648d1b8d99af0b60c3659c3d4da33ac819ee007b Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 13 Oct 2017 09:49:15 -0600 Subject: [PATCH 120/184] Catch CommandExecutionError --- salt/modules/win_lgpo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 7d2998c272..c283a0b346 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -2873,7 +2873,7 @@ def _findOptionValueInSeceditFile(option): for _ in range(5): try: __salt__['file.remove'](_tfile) - except WindowsError: # pylint: disable=E0602 + except CommandExecutionError: time.sleep(.1) continue else: From 5230ecd7e1da1c1d591e64998bc1f56a1ba96bbe Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 12 Oct 2017 15:43:23 -0600 Subject: [PATCH 121/184] Accept *args --- salt/modules/win_groupadd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/win_groupadd.py b/salt/modules/win_groupadd.py index f584a0ab94..4fb71d4061 100644 --- a/salt/modules/win_groupadd.py +++ b/salt/modules/win_groupadd.py @@ -36,7 +36,7 @@ def __virtual__(): return (False, "Module win_groupadd: module only works on Windows systems") -def add(name, **kwargs): +def add(name, *args, **kwargs): ''' Add the specified group From aa278966de4717025c698483b4dd08c550caaaea Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 12 Oct 2017 16:00:40 -0600 Subject: [PATCH 122/184] Remove *args, pass gid as a keyword --- salt/modules/win_groupadd.py | 2 +- salt/states/group.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/salt/modules/win_groupadd.py b/salt/modules/win_groupadd.py index 4fb71d4061..f584a0ab94 100644 --- a/salt/modules/win_groupadd.py +++ b/salt/modules/win_groupadd.py @@ -36,7 +36,7 @@ def __virtual__(): return (False, "Module win_groupadd: module only works on Windows systems") -def add(name, *args, **kwargs): +def add(name, **kwargs): ''' Add the specified group diff --git a/salt/states/group.py b/salt/states/group.py index d280243e08..8ba25ab513 100644 --- a/salt/states/group.py +++ b/salt/states/group.py @@ -244,9 +244,7 @@ def present(name, return ret # Group is not present, make it. - if __salt__['group.add'](name, - gid, - system=system): + if __salt__['group.add'](name, gid=gid, system=system): # if members to be added grp_members = None if members: From 83f36cc2efef174b2a118ea39e3de2c3cd68f1c9 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 12 Oct 2017 16:36:37 -0600 Subject: [PATCH 123/184] Account for 15 character limit in hostname --- salt/utils/win_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/win_functions.py b/salt/utils/win_functions.py index 4e3ec9663c..c39a546f9d 100644 --- a/salt/utils/win_functions.py +++ b/salt/utils/win_functions.py @@ -154,6 +154,6 @@ def get_sam_name(username): i.e. salt.utils.fix_local_user('Administrator') would return 'computername\administrator' ''' if '\\' not in username: - username = '{0}\\{1}'.format(platform.node(), username) + username = '{0}\\{1}'.format(platform.node()[:15], username) return username.lower() From 43740c5fed120413e9b439bfc547330d8472e76e Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 12 Oct 2017 16:39:11 -0600 Subject: [PATCH 124/184] Document 15 character limit --- salt/utils/win_functions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/utils/win_functions.py b/salt/utils/win_functions.py index c39a546f9d..85a703aae3 100644 --- a/salt/utils/win_functions.py +++ b/salt/utils/win_functions.py @@ -152,6 +152,8 @@ def get_sam_name(username): Everything is returned lower case i.e. salt.utils.fix_local_user('Administrator') would return 'computername\administrator' + + .. note:: Long computer names are truncated to 15 characters ''' if '\\' not in username: username = '{0}\\{1}'.format(platform.node()[:15], username) From ef759a3875c55cb46fac9e8326dae3fb53721a30 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 17 Oct 2017 11:32:53 -0600 Subject: [PATCH 125/184] Fix example in function docs for get_sam_name --- salt/utils/win_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/win_functions.py b/salt/utils/win_functions.py index 85a703aae3..1ef6122637 100644 --- a/salt/utils/win_functions.py +++ b/salt/utils/win_functions.py @@ -151,7 +151,7 @@ def get_sam_name(username): Everything is returned lower case - i.e. salt.utils.fix_local_user('Administrator') would return 'computername\administrator' + i.e. salt.utils.get_same_name('Administrator') would return 'computername\administrator' .. note:: Long computer names are truncated to 15 characters ''' From 29bc80ff87bd4fb70295609247df2862b007ba2f Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 17 Oct 2017 16:44:44 -0600 Subject: [PATCH 126/184] Improve get_sam_name Return the actual sam name as reported by LookupAccountName If the user does not exist, return a fabricated SAM name Move lower to the state module --- salt/modules/win_groupadd.py | 4 ++-- salt/states/group.py | 8 ++++---- salt/utils/win_functions.py | 20 ++++++++++---------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/salt/modules/win_groupadd.py b/salt/modules/win_groupadd.py index f584a0ab94..17d0ffdffb 100644 --- a/salt/modules/win_groupadd.py +++ b/salt/modules/win_groupadd.py @@ -250,7 +250,7 @@ def adduser(name, username, **kwargs): '/', '\\').encode('ascii', 'backslashreplace').lower()) try: - if salt.utils.win_functions.get_sam_name(username) not in existingMembers: + if salt.utils.win_functions.get_sam_name(username).lower() not in existingMembers: if not __opts__['test']: groupObj.Add('WinNT://' + username.replace('\\', '/')) @@ -309,7 +309,7 @@ def deluser(name, username, **kwargs): '/', '\\').encode('ascii', 'backslashreplace').lower()) try: - if salt.utils.win_functions.get_sam_name(username) in existingMembers: + if salt.utils.win_functions.get_sam_name(username).lower() in existingMembers: if not __opts__['test']: groupObj.Remove('WinNT://' + username.replace('\\', '/')) diff --git a/salt/states/group.py b/salt/states/group.py index 8ba25ab513..dab2b5bf18 100644 --- a/salt/states/group.py +++ b/salt/states/group.py @@ -65,11 +65,11 @@ def _changes(name, if lgrp['members']: lgrp['members'] = [user.lower() for user in lgrp['members']] if members: - members = [salt.utils.win_functions.get_sam_name(user) for user in members] + members = [salt.utils.win_functions.get_sam_name(user).lower() for user in members] if addusers: - addusers = [salt.utils.win_functions.get_sam_name(user) for user in addusers] + addusers = [salt.utils.win_functions.get_sam_name(user).lower() for user in addusers] if delusers: - delusers = [salt.utils.win_functions.get_sam_name(user) for user in delusers] + delusers = [salt.utils.win_functions.get_sam_name(user).lower() for user in delusers] change = {} if gid: @@ -267,7 +267,7 @@ def present(name, ret['result'] = False ret['comment'] = ( 'Group {0} has been created but, some changes could not' - ' be applied') + ' be applied'.format(name)) ret['changes'] = {'Failed': changes} else: ret['result'] = False diff --git a/salt/utils/win_functions.py b/salt/utils/win_functions.py index 1ef6122637..efaac09f2e 100644 --- a/salt/utils/win_functions.py +++ b/salt/utils/win_functions.py @@ -111,7 +111,7 @@ def get_sid_from_name(name): sid = win32security.LookupAccountName(None, name)[0] except pywintypes.error as exc: raise CommandExecutionError( - 'User {0} found: {1}'.format(name, exc.strerror)) + 'User {0} not found: {1}'.format(name, exc.strerror)) return win32security.ConvertSidToStringSid(sid) @@ -146,16 +146,16 @@ def get_current_user(): def get_sam_name(username): ''' Gets the SAM name for a user. It basically prefixes a username without a - backslash with the computer name. If the username contains a backslash, it - is returned as is. + backslash with the computer name. If the user does not exist, a SAM + compatible name will be returned using the local hostname as the domain. - Everything is returned lower case - - i.e. salt.utils.get_same_name('Administrator') would return 'computername\administrator' + i.e. salt.utils.get_same_name('Administrator') would return 'DOMAIN.COM\Administrator' .. note:: Long computer names are truncated to 15 characters ''' - if '\\' not in username: - username = '{0}\\{1}'.format(platform.node()[:15], username) - - return username.lower() + try: + sid_obj = win32security.LookupAccountName(None, username)[0] + except pywintypes.error: + return '\\'.join([platform.node()[:15].upper(), username]) + username, domain, _ = win32security.LookupAccountSid(None, sid_obj) + return '\\'.join([domain, username]) From 6da4fd979bd736ec201d965b2a685a8062ab4d16 Mon Sep 17 00:00:00 2001 From: Eric Radman Date: Thu, 12 Oct 2017 09:49:41 -0400 Subject: [PATCH 127/184] Check file attributes in check_file_meta() if lsattr(1) is installed lsattr/chattr is not installed on many Unix-like platforms by default, including *BSD, Solaris, and minimal Linux distributions such as Alpine. --- salt/modules/file.py | 16 +++++++++------- tests/unit/modules/test_file.py | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index eb8327a187..17d0e253d8 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -4700,6 +4700,7 @@ def check_file_meta( contents File contents ''' + lsattr_cmd = salt.utils.path.which('lsattr') changes = {} if not source_sum: source_sum = {} @@ -4764,13 +4765,14 @@ def check_file_meta( if mode is not None and mode != smode: changes['mode'] = mode - diff_attrs = _cmp_attrs(name, attrs) - if ( - attrs is not None and - diff_attrs[0] is not None or - diff_attrs[1] is not None - ): - changes['attrs'] = attrs + if lsattr_cmd: + diff_attrs = _cmp_attrs(name, attrs) + if ( + attrs is not None and + diff_attrs[0] is not None or + diff_attrs[1] is not None + ): + changes['attrs'] = attrs return changes diff --git a/tests/unit/modules/test_file.py b/tests/unit/modules/test_file.py index 540c805ff5..4547a17dca 100644 --- a/tests/unit/modules/test_file.py +++ b/tests/unit/modules/test_file.py @@ -504,6 +504,26 @@ class FileModuleTestCase(TestCase, LoaderModuleMockMixin): } } + def test_check_file_meta_no_lsattr(self): + ''' + Ensure that we skip attribute comparison if lsattr(1) is not found + ''' + source = "salt:///README.md" + name = "/home/git/proj/a/README.md" + source_sum = {} + stats_result = {'size': 22, 'group': 'wheel', 'uid': 0, 'type': 'file', + 'mode': '0600', 'gid': 0, 'target': name, 'user': + 'root', 'mtime': 1508356390, 'atime': 1508356390, + 'inode': 447, 'ctime': 1508356390} + with patch('salt.modules.file.stats') as m_stats: + m_stats.return_value = stats_result + with patch('salt.utils.path.which') as m_which: + m_which.return_value = None + result = filemod.check_file_meta(name, name, source, source_sum, + 'root', 'root', '755', None, + 'base') + self.assertTrue(result, None) + @skipIf(salt.utils.platform.is_windows(), 'SED is not available on Windows') def test_sed_limit_escaped(self): with tempfile.NamedTemporaryFile(mode='w+') as tfile: From 2551454f6aeb062a3d1141f4b89837a62a7e40d4 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Thu, 19 Oct 2017 18:26:08 -0500 Subject: [PATCH 128/184] Add absolute_import --- tests/unit/utils/test_xmlutil.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/utils/test_xmlutil.py b/tests/unit/utils/test_xmlutil.py index 2377ad782c..b17b0af3e6 100644 --- a/tests/unit/utils/test_xmlutil.py +++ b/tests/unit/utils/test_xmlutil.py @@ -3,6 +3,7 @@ tests.unit.xmlutil_test ~~~~~~~~~~~~~~~~~~~~ ''' +from __future__ import absolute_import # Import Salt Testing libs from tests.support.unit import TestCase From d3769b3568659831f90a657e9b7cdb818b599466 Mon Sep 17 00:00:00 2001 From: Vernon Cole Date: Thu, 19 Oct 2017 18:50:53 -0600 Subject: [PATCH 129/184] add saltify wake-on-lan and document it --- doc/topics/cloud/misc.rst | 2 + doc/topics/cloud/saltify.rst | 7 +++- salt/cloud/clouds/saltify.py | 54 ++++++++++++++++++++++++- tests/unit/cloud/clouds/test_saltify.py | 37 +++++++++++++++-- 4 files changed, 95 insertions(+), 5 deletions(-) diff --git a/doc/topics/cloud/misc.rst b/doc/topics/cloud/misc.rst index 2e44aa5612..b485d28909 100644 --- a/doc/topics/cloud/misc.rst +++ b/doc/topics/cloud/misc.rst @@ -1,3 +1,5 @@ +.. _misc-salt-cloud-options: + ================================ Miscellaneous Salt Cloud Options ================================ diff --git a/doc/topics/cloud/saltify.rst b/doc/topics/cloud/saltify.rst index 4f1b41546d..fb3662ab02 100644 --- a/doc/topics/cloud/saltify.rst +++ b/doc/topics/cloud/saltify.rst @@ -37,6 +37,12 @@ the role usually performed by a vendor's cloud management system. The salt maste must be running on the salt-cloud machine, and created nodes must be connected to the master. +Additional information about which configuration options apply to which actions +can be studied in the +:ref:`Saltify Module documentation ` +and the +:ref:`Miscellaneous Salt Cloud Options ` +document. Profiles ======== @@ -77,7 +83,6 @@ to it can be verified with Salt: salt my-machine test.ping - Destroy Options --------------- diff --git a/salt/cloud/clouds/saltify.py b/salt/cloud/clouds/saltify.py index 3215d8a32c..a324d33723 100644 --- a/salt/cloud/clouds/saltify.py +++ b/salt/cloud/clouds/saltify.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- ''' +.. _`saltify-module`: + Saltify Module ============== @@ -7,6 +9,9 @@ The Saltify module is designed to install Salt on a remote machine, virtual or bare metal, using SSH. This module is useful for provisioning machines which are already installed, but not Salted. +.. versionchanged:: Oxygen + The wake_on_lan capability, and actions destroy, reboot, and query functions were added. + Use of this module requires some configuration in cloud profile and provider files as described in the :ref:`Gettting Started with Saltify ` documentation. @@ -15,6 +20,7 @@ files as described in the # Import python libs from __future__ import absolute_import import logging +import time # Import salt libs import salt.utils.cloud @@ -210,12 +216,58 @@ def show_instance(name, call=None): def create(vm_): ''' - Provision a single machine + if configuration parameter ``deploy`` is ``True``, + + Provision a single machine, adding its keys to the salt master + + else, + + Test ssh connections to the machine + + Configuration parameters: + + - deploy: (see above) + - provider: name of entry in ``salt/cloud.providers.d/???`` file + - ssh_host: IP address or DNS name of the new machine + - ssh_username: name used to log in to the new machine + - ssh_password: password to log in (unless key_filename is used) + - key_filename: (optional) SSH private key for passwordless login + - ssh_port: (default=22) TCP port for SSH connection + - wake_on_lan_mac: (optional) hardware (MAC) address for wake on lan + - wol_sender_node: (optional) salt minion to send wake on lan command + - wol_boot_wait: (default=30) seconds to delay while client boots + - force_minion_config: (optional) replace the minion configuration files on the new machine + + See also + :ref:`Miscellaneous Salt Cloud Options ` + and + :ref:`Getting Started with Saltify ` + + CLI Example: + + .. code-block:: bash + + salt-cloud -p mymachine my_new_id ''' deploy_config = config.get_cloud_config_value( 'deploy', vm_, __opts__, default=False) if deploy_config: + wol_mac = config.get_cloud_config_value( + 'wake_on_lan_mac', vm_, __opts__, default='') + wol_host = config.get_cloud_config_value( + 'wol_sender_node', vm_, __opts__, default = '') + if wol_mac and wol_host: + log.info('sending wake-on-lan to %s using node %s', + wol_mac, wol_host) + local = salt.client.LocalClient() + ret = local.cmd(wol_host, 'network.wol', [wol_mac]) + log.info('network.wol returned value %s', ret) + if ret and ret[wol_host]: + sleep_time = config.get_cloud_config_value( + 'wol_boot_wait', vm_, __opts__, default = 30) + log.info('delaying %d seconds for boot', sleep_time) + time.sleep(sleep_time) log.info('Provisioning existing machine %s', vm_['name']) ret = __utils__['cloud.bootstrap'](vm_, __opts__) else: diff --git a/tests/unit/cloud/clouds/test_saltify.py b/tests/unit/cloud/clouds/test_saltify.py index dad6673ded..4ca83e346c 100644 --- a/tests/unit/cloud/clouds/test_saltify.py +++ b/tests/unit/cloud/clouds/test_saltify.py @@ -22,9 +22,14 @@ TEST_PROFILES = { 'ssh_username': 'fred', 'remove_config_on_destroy': False, # expected for test 'shutdown_on_destroy': True # expected value for test - } + }, + 'testprofile3': { # this profile is used in test_create_wake_on_lan() + 'wake_on_lan_mac': 'aa-bb-cc-dd-ee-ff', + 'wol_sender_node': 'friend1', + 'wol_boot_wait': 0.01 # we want the wait to be very short + } } -TEST_PROFILE_NAMES = ['testprofile1', 'testprofile2'] +TEST_PROFILE_NAMES = ['testprofile1', 'testprofile2', 'testprofile3'] @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -78,12 +83,38 @@ class SaltifyTestCase(TestCase, LoaderModuleMockMixin): {'cloud.bootstrap': mock_cmd}): vm_ = {'deploy': True, 'driver': 'saltify', - 'name': 'testprofile2', + 'name': 'new2', + 'profile': 'testprofile2', } result = saltify.create(vm_) mock_cmd.assert_called_once_with(vm_, ANY) self.assertTrue(result) + def test_create_wake_on_lan(self): + ''' + Test if wake on lan works + ''' + mock_sleep = MagicMock() + mock_cmd = MagicMock(return_value=True) + mm_cmd = MagicMock(return_value={'friend1': True}) + lcl = salt.client.LocalClient() + lcl.cmd = mm_cmd + with patch('time.sleep', mock_sleep): + with patch('salt.client.LocalClient', return_value=lcl): + with patch.dict( + 'salt.cloud.clouds.saltify.__utils__', + {'cloud.bootstrap': mock_cmd}): + vm_ = {'deploy': True, + 'driver': 'saltify', + 'name': 'new1', + 'profile': 'testprofile3', + } + result = saltify.create(vm_) + mock_cmd.assert_called_once_with(vm_, ANY) + mm_cmd.assert_called_with('friend1', 'network.wol', ['aa-bb-cc-dd-ee-ff']) + mock_sleep.assert_called_with(0.01) + self.assertTrue(result) + def test_avail_locations(self): ''' Test the avail_locations will always return {} From cea457efa01929856ed79c3fd4598d340ed60cad Mon Sep 17 00:00:00 2001 From: Vernon Cole Date: Thu, 19 Oct 2017 20:12:01 -0600 Subject: [PATCH 130/184] more better documentation, fix parameter handling --- doc/topics/cloud/saltify.rst | 36 ++++++++++++++++++++++++++++++++++++ salt/cloud/clouds/saltify.py | 9 ++++++--- salt/cloud/clouds/vagrant.py | 2 +- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/doc/topics/cloud/saltify.rst b/doc/topics/cloud/saltify.rst index fb3662ab02..20e08deb33 100644 --- a/doc/topics/cloud/saltify.rst +++ b/doc/topics/cloud/saltify.rst @@ -107,6 +107,42 @@ and will attempt the following options: .. versionadded:: Oxygen +Wake On LAN +----------- +In addition to connecting a hardware machine to a Salt master, +you have the option of sending a wake-on-LAN +`magic packet`_ +to start the machine running. + +.. _magic packet: https://en.wikipedia.org/wiki/Wake-on-LAN + +The "magic packet" must be sent by an existing salt minion which is on +the same network segment as the target machine. (Or your router +must be set up especially to route WoL packets.) Your target machine +must be set up to listen for WoL and to respond appropriatly. + +You must provide the Salt node id of the machine which will send +the WoL packet \(parameter ``wol_sender_node``\), and +the hardware MAC address of the machine you intend to wake, +\(parameter ``wake_on_lan_mac``\). If both parameters are defined, +the WoL will be sent. The cloud master will then sleep a while +\(parameter ``wol_boot_wait``) to give the target machine time to +boot up before we start probing its SSH port to begin deploying +Salt to it. The default sleep time is 30 seconds. + +.. code-block:: yaml + + # /etc/salt/cloud.profiles.d/saltify.conf + + salt-this-machine: + ssh_host: 12.34.56.78 + ssh_username: root + key_filename: '/etc/salt/mysshkey.pem' + provider: my-saltify-config + wake_on_lan_mac: '00:e0:4c:70:2a:b2' # found with ifconfig + wol_sender_node: bevymaster # its on this network segment + wol_boot_wait: 45 # seconds to sleep + Using Map Files --------------- The settings explained in the section above may also be set in a map file. An diff --git a/salt/cloud/clouds/saltify.py b/salt/cloud/clouds/saltify.py index a324d33723..d3b6ef82c1 100644 --- a/salt/cloud/clouds/saltify.py +++ b/salt/cloud/clouds/saltify.py @@ -261,13 +261,16 @@ def create(vm_): log.info('sending wake-on-lan to %s using node %s', wol_mac, wol_host) local = salt.client.LocalClient() - ret = local.cmd(wol_host, 'network.wol', [wol_mac]) + if isinstance(wol_mac, six.string_types): + wol_mac = [wol_mac] # a smart user may have passed more params + ret = local.cmd(wol_host, 'network.wol', wol_mac) log.info('network.wol returned value %s', ret) if ret and ret[wol_host]: sleep_time = config.get_cloud_config_value( 'wol_boot_wait', vm_, __opts__, default = 30) - log.info('delaying %d seconds for boot', sleep_time) - time.sleep(sleep_time) + if sleep_time > 0.0: + log.info('delaying %d seconds for boot', sleep_time) + time.sleep(sleep_time) log.info('Provisioning existing machine %s', vm_['name']) ret = __utils__['cloud.bootstrap'](vm_, __opts__) else: diff --git a/salt/cloud/clouds/vagrant.py b/salt/cloud/clouds/vagrant.py index 08fec40997..830c8b57da 100644 --- a/salt/cloud/clouds/vagrant.py +++ b/salt/cloud/clouds/vagrant.py @@ -8,7 +8,7 @@ Salt minion. Use of this module requires some configuration in cloud profile and provider files as described in the -:ref:`Gettting Started with Vagrant ` documentation. +:ref:`Getting Started with Vagrant ` documentation. .. versionadded:: Oxygen From 9afc2e94df9e38a2cba4fe09b23e9af5d63f3612 Mon Sep 17 00:00:00 2001 From: Sergey Kacheev Date: Fri, 20 Oct 2017 16:20:23 +0700 Subject: [PATCH 131/184] pass "verify" from runner to make_request --- salt/runners/vault.py | 6 +++++- salt/utils/vault.py | 9 +++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/salt/runners/vault.py b/salt/runners/vault.py index e35aba2f67..d494e8e827 100644 --- a/salt/runners/vault.py +++ b/salt/runners/vault.py @@ -65,7 +65,11 @@ def generate_token(minion_id, signature, impersonated_by_master=False): return {'error': response.reason} authData = response.json()['auth'] - return {'token': authData['client_token'], 'url': config['url']} + return { + 'token': authData['client_token'], + 'url': config['url'], + 'verify': verify, + } except Exception as e: return {'error': str(e)} diff --git a/salt/utils/vault.py b/salt/utils/vault.py index e659851f22..fef7611209 100644 --- a/salt/utils/vault.py +++ b/salt/utils/vault.py @@ -90,7 +90,8 @@ def _get_token_and_url_from_master(): raise salt.exceptions.CommandExecutionError(result) return { 'url': result['url'], - 'token': result['token'] + 'token': result['token'], + 'verify': result['verify'], } @@ -104,7 +105,8 @@ def _get_vault_connection(): try: return { 'url': __opts__['vault']['url'], - 'token': __opts__['vault']['auth']['token'] + 'token': __opts__['vault']['auth']['token'], + 'verify': __opts__['vault'].get('verify', None) } except KeyError as err: errmsg = 'Minion has "vault" config section, but could not find key "{0}" within'.format(err.message) @@ -124,9 +126,8 @@ def make_request(method, resource, profile=None, **args): connection = _get_vault_connection() token, vault_url = connection['token'], connection['url'] - if "verify" not in args: - args["verify"] = __opts__['vault'].get('verify', None) + args["verify"] = connection['verify'] url = "{0}/{1}".format(vault_url, resource) headers = {'X-Vault-Token': token, 'Content-Type': 'application/json'} From 8193b2f32a2733bf5686d8daf95a64d8b4de96f6 Mon Sep 17 00:00:00 2001 From: Oleg Iurchenko Date: Fri, 20 Oct 2017 09:39:28 +0000 Subject: [PATCH 132/184] Add client version discovery to Keystone module This patch adds keystoneclient.discover.Discover feature to Keystone module --- salt/modules/keystone.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/salt/modules/keystone.py b/salt/modules/keystone.py index ecd061deaf..c4d12085eb 100644 --- a/salt/modules/keystone.py +++ b/salt/modules/keystone.py @@ -61,10 +61,11 @@ from salt.ext import six HAS_KEYSTONE = False try: # pylint: disable=import-error - from keystoneclient.v2_0 import client import keystoneclient.exceptions HAS_KEYSTONE = True - from keystoneclient.v3 import client as client3 + from keystoneclient import discover + from keystoneauth1 import session + from keystoneauth1.identity import generic # pylint: enable=import-error except ImportError: pass @@ -111,6 +112,9 @@ def _get_kwargs(profile=None, **connection_args): insecure = get('insecure', False) token = get('token') endpoint = get('endpoint', 'http://127.0.0.1:35357/v2.0') + user_domain_name=get('user_domain_name', 'Default') + project_domain_name=get('project_domain_name', 'Default') + if token: kwargs = {'token': token, @@ -120,7 +124,9 @@ def _get_kwargs(profile=None, **connection_args): 'password': password, 'tenant_name': tenant, 'tenant_id': tenant_id, - 'auth_url': auth_url} + 'auth_url': auth_url, + 'user_domain_name': user_domain_name, + 'project_domain_name': project_domain_name} # 'insecure' keyword not supported by all v2.0 keystone clients # this ensures it's only passed in when defined if insecure: @@ -159,15 +165,23 @@ def auth(profile=None, **connection_args): ''' kwargs = _get_kwargs(profile=profile, **connection_args) - if float(api_version(profile=profile, **connection_args).strip('v')) >= 3: + disc = discover.Discover(auth_url=kwargs['auth_url']) + v2_auth_url = disc.url_for('v2.0') + v3_auth_url = disc.url_for('v3.0') + if v3_auth_url: global _OS_IDENTITY_API_VERSION global _TENANTS _OS_IDENTITY_API_VERSION = 3 _TENANTS = 'projects' - return client3.Client(**kwargs) + kwargs['auth_url']=v3_auth_url else: - return client.Client(**kwargs) - + kwargs['auth_url']=v2_auth_url + kwargs.pop('user_domain_name') + kwargs.pop('project_domain_name') + auth = generic.Password(**kwargs) + sess = session.Session(auth=auth) + ks_cl = disc.create_client(session=sess) + return ks_cl def ec2_credentials_create(user_id=None, name=None, tenant_id=None, tenant=None, From 6580c6d28f1f3b971919dbc08fa3fc649fcd724a Mon Sep 17 00:00:00 2001 From: Vernon Cole Date: Fri, 20 Oct 2017 08:03:22 -0600 Subject: [PATCH 133/184] lint fix --- salt/cloud/clouds/saltify.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/cloud/clouds/saltify.py b/salt/cloud/clouds/saltify.py index d3b6ef82c1..b4aa137dc1 100644 --- a/salt/cloud/clouds/saltify.py +++ b/salt/cloud/clouds/saltify.py @@ -256,7 +256,7 @@ def create(vm_): wol_mac = config.get_cloud_config_value( 'wake_on_lan_mac', vm_, __opts__, default='') wol_host = config.get_cloud_config_value( - 'wol_sender_node', vm_, __opts__, default = '') + 'wol_sender_node', vm_, __opts__, default='') if wol_mac and wol_host: log.info('sending wake-on-lan to %s using node %s', wol_mac, wol_host) @@ -267,7 +267,7 @@ def create(vm_): log.info('network.wol returned value %s', ret) if ret and ret[wol_host]: sleep_time = config.get_cloud_config_value( - 'wol_boot_wait', vm_, __opts__, default = 30) + 'wol_boot_wait', vm_, __opts__, default=30) if sleep_time > 0.0: log.info('delaying %d seconds for boot', sleep_time) time.sleep(sleep_time) From f3f30907948a47556d9630b6df21f9aabd2e8910 Mon Sep 17 00:00:00 2001 From: Oleg Iurchenko Date: Fri, 20 Oct 2017 14:35:26 +0000 Subject: [PATCH 134/184] Fix Salt CI issues --- salt/modules/keystone.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/salt/modules/keystone.py b/salt/modules/keystone.py index c4d12085eb..a82a5b46b7 100644 --- a/salt/modules/keystone.py +++ b/salt/modules/keystone.py @@ -61,8 +61,10 @@ from salt.ext import six HAS_KEYSTONE = False try: # pylint: disable=import-error + from keystoneclient.v2_0 import client import keystoneclient.exceptions HAS_KEYSTONE = True + from keystoneclient.v3 import client as client3 from keystoneclient import discover from keystoneauth1 import session from keystoneauth1.identity import generic @@ -112,10 +114,8 @@ def _get_kwargs(profile=None, **connection_args): insecure = get('insecure', False) token = get('token') endpoint = get('endpoint', 'http://127.0.0.1:35357/v2.0') - user_domain_name=get('user_domain_name', 'Default') - project_domain_name=get('project_domain_name', 'Default') - - + user_domain_name = get('user_domain_name', 'Default') + project_domain_name = get('project_domain_name', 'Default') if token: kwargs = {'token': token, 'endpoint': endpoint} @@ -173,9 +173,9 @@ def auth(profile=None, **connection_args): global _TENANTS _OS_IDENTITY_API_VERSION = 3 _TENANTS = 'projects' - kwargs['auth_url']=v3_auth_url + kwargs['auth_url'] = v3_auth_url else: - kwargs['auth_url']=v2_auth_url + kwargs['auth_url'] = v2_auth_url kwargs.pop('user_domain_name') kwargs.pop('project_domain_name') auth = generic.Password(**kwargs) @@ -183,6 +183,7 @@ def auth(profile=None, **connection_args): ks_cl = disc.create_client(session=sess) return ks_cl + def ec2_credentials_create(user_id=None, name=None, tenant_id=None, tenant=None, profile=None, **connection_args): From 3b9d02977d361b750395d53557158d36968fd10b Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 20 Oct 2017 10:48:37 -0400 Subject: [PATCH 135/184] Update deprecated salt.utils paths to new salt.utils.x paths --- tests/integration/spm/test_build.py | 6 +++--- tests/unit/test_auth.py | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/integration/spm/test_build.py b/tests/integration/spm/test_build.py index 79743e0187..18eb931ad3 100644 --- a/tests/integration/spm/test_build.py +++ b/tests/integration/spm/test_build.py @@ -13,7 +13,7 @@ from tests.support.case import SPMCase from tests.support.helpers import destructiveTest # Import Salt Libraries -import salt.utils +import salt.utils.files @destructiveTest @@ -32,14 +32,14 @@ class SPMBuildTest(SPMCase): for formula_dir in dirs: os.makedirs(formula_dir) - with salt.utils.fopen(self.formula_sls, 'w') as fp: + with salt.utils.files.fopen(self.formula_sls, 'w') as fp: fp.write(textwrap.dedent('''\ install-apache: pkg.installed: - name: apache2 ''')) - with salt.utils.fopen(self.formula_file, 'w') as fp: + with salt.utils.files.fopen(self.formula_file, 'w') as fp: fp.write(textwrap.dedent('''\ name: apache os: RedHat, Debian, Ubuntu, Suse, FreeBSD diff --git a/tests/unit/test_auth.py b/tests/unit/test_auth.py index 03da1b6550..5b0d9643e7 100644 --- a/tests/unit/test_auth.py +++ b/tests/unit/test_auth.py @@ -14,7 +14,7 @@ from tests.support.mock import patch, call, NO_MOCK, NO_MOCK_REASON, MagicMock import salt.master from tests.support.case import ModuleCase from salt import auth -import salt.utils +import salt.utils.platform @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -151,7 +151,7 @@ class MasterACLTestCase(ModuleCase): } self.addCleanup(delattr, self, 'valid_clear_load') - @skipIf(salt.utils.is_windows(), 'PAM eauth not available on Windows') + @skipIf(salt.utils.platform.is_windows(), 'PAM eauth not available on Windows') def test_master_publish_name(self): ''' Test to ensure a simple name can auth against a given function. @@ -222,7 +222,7 @@ class MasterACLTestCase(ModuleCase): self.clear.publish(self.valid_clear_load) self.assertEqual(self.fire_event_mock.mock_calls, []) - @skipIf(salt.utils.is_windows(), 'PAM eauth not available on Windows') + @skipIf(salt.utils.platform.is_windows(), 'PAM eauth not available on Windows') def test_master_minion_glob(self): ''' Test to ensure we can allow access to a given @@ -260,7 +260,7 @@ class MasterACLTestCase(ModuleCase): # Unimplemented pass - @skipIf(salt.utils.is_windows(), 'PAM eauth not available on Windows') + @skipIf(salt.utils.platform.is_windows(), 'PAM eauth not available on Windows') def test_args_empty_spec(self): ''' Test simple arg restriction allowed. @@ -279,7 +279,7 @@ class MasterACLTestCase(ModuleCase): self.clear.publish(self.valid_clear_load) self.assertEqual(self.fire_event_mock.call_args[0][0]['fun'], 'test.empty') - @skipIf(salt.utils.is_windows(), 'PAM eauth not available on Windows') + @skipIf(salt.utils.platform.is_windows(), 'PAM eauth not available on Windows') def test_args_simple_match(self): ''' Test simple arg restriction allowed. @@ -301,7 +301,7 @@ class MasterACLTestCase(ModuleCase): self.clear.publish(self.valid_clear_load) self.assertEqual(self.fire_event_mock.call_args[0][0]['fun'], 'test.echo') - @skipIf(salt.utils.is_windows(), 'PAM eauth not available on Windows') + @skipIf(salt.utils.platform.is_windows(), 'PAM eauth not available on Windows') def test_args_more_args(self): ''' Test simple arg restriction allowed to pass unlisted args. @@ -362,7 +362,7 @@ class MasterACLTestCase(ModuleCase): self.clear.publish(self.valid_clear_load) self.assertEqual(self.fire_event_mock.mock_calls, []) - @skipIf(salt.utils.is_windows(), 'PAM eauth not available on Windows') + @skipIf(salt.utils.platform.is_windows(), 'PAM eauth not available on Windows') def test_args_kwargs_match(self): ''' Test simple kwargs restriction allowed. @@ -436,7 +436,7 @@ class MasterACLTestCase(ModuleCase): self.clear.publish(self.valid_clear_load) self.assertEqual(self.fire_event_mock.mock_calls, []) - @skipIf(salt.utils.is_windows(), 'PAM eauth not available on Windows') + @skipIf(salt.utils.platform.is_windows(), 'PAM eauth not available on Windows') def test_args_mixed_match(self): ''' Test mixed args and kwargs restriction allowed. @@ -582,7 +582,7 @@ class AuthACLTestCase(ModuleCase): } self.addCleanup(delattr, self, 'valid_clear_load') - @skipIf(salt.utils.is_windows(), 'PAM eauth not available on Windows') + @skipIf(salt.utils.platform.is_windows(), 'PAM eauth not available on Windows') def test_acl_simple_allow(self): self.clear.publish(self.valid_clear_load) self.assertEqual(self.auth_check_mock.call_args[0][0], From 9da66a2bb97814afc43f3ffa7ab15e7bab9bfcee Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 20 Oct 2017 10:53:11 -0400 Subject: [PATCH 136/184] Reduce the number of days an issue is stale by 25 --- .github/stale.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/stale.yml b/.github/stale.yml index 2aa60bdc61..754018e841 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -1,8 +1,8 @@ # Probot Stale configuration file # Number of days of inactivity before an issue becomes stale -# 950 is approximately 2 years and 7 months -daysUntilStale: 950 +# 925 is approximately 2 years and 6 months +daysUntilStale: 925 # Number of days of inactivity before a stale issue is closed daysUntilClose: 7 From c830f0f8f4db519aca24d359e7594b5a838e1aa7 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 20 Oct 2017 10:59:37 -0400 Subject: [PATCH 137/184] Add some space in new opsgenie docs for rst formmating Follow up to #43821 These fixes will help the docs render correctly with the various examples given. --- salt/modules/opsgenie.py | 6 ++++++ salt/states/opsgenie.py | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/salt/modules/opsgenie.py b/salt/modules/opsgenie.py index cfde15b90e..acfb6d37e5 100644 --- a/salt/modules/opsgenie.py +++ b/salt/modules/opsgenie.py @@ -6,8 +6,11 @@ Module for sending data to OpsGenie :configuration: This module can be used in Reactor System for posting data to OpsGenie as a remote-execution function. + For example: + .. code-block:: yaml + opsgenie_event_poster: local.opsgenie.post_data: - tgt: 'salt-minion' @@ -40,6 +43,9 @@ def post_data(api_key=None, name='OpsGenie Execution Module', reason=None, module with your designated tag (og-tag in this case). CLI Example: + + .. code-block:: bash + salt-call event.send 'og-tag' '{"reason" : "Overheating CPU!"}' Required parameters: diff --git a/salt/states/opsgenie.py b/salt/states/opsgenie.py index d86c32ebb6..cf129b1335 100644 --- a/salt/states/opsgenie.py +++ b/salt/states/opsgenie.py @@ -2,10 +2,14 @@ ''' Create/Close an alert in OpsGenie ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + .. versionadded:: Oxygen + This state is useful for creating or closing alerts in OpsGenie during state runs. + .. code-block:: yaml + used_space: disk.status: - name: / From b7c0182d93a1092b7369eedfbcf5bc2512c12f1b Mon Sep 17 00:00:00 2001 From: Vernon Cole Date: Fri, 20 Oct 2017 09:24:26 -0600 Subject: [PATCH 138/184] correct and clarify documentation --- doc/topics/cloud/saltify.rst | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/doc/topics/cloud/saltify.rst b/doc/topics/cloud/saltify.rst index 20e08deb33..a0f058cd4d 100644 --- a/doc/topics/cloud/saltify.rst +++ b/doc/topics/cloud/saltify.rst @@ -4,7 +4,7 @@ Getting Started With Saltify ============================ -The Saltify driver is a new, experimental driver for installing Salt on existing +The Saltify driver is a driver for installing Salt on existing machines (virtual or bare metal). @@ -47,12 +47,15 @@ document. Profiles ======== -Saltify requires a profile to be configured for each machine that needs Salt -installed. The initial profile can be set up at ``/etc/salt/cloud.profiles`` +Saltify requires a separate profile to be configured for each machine that +needs Salt installed [#]_. The initial profile can be set up at +``/etc/salt/cloud.profiles`` or in the ``/etc/salt/cloud.profiles.d/`` directory. Each profile requires both an ``ssh_host`` and an ``ssh_username`` key parameter as well as either an ``key_filename`` or a ``password``. +.. [#] Unless you are using a map file to provide the unique parameters. + Profile configuration example: .. code-block:: yaml @@ -74,45 +77,48 @@ The machine can now be "Salted" with the following command: This will install salt on the machine specified by the cloud profile, ``salt-this-machine``, and will give the machine the minion id of ``my-machine``. If the command was executed on the salt-master, its Salt -key will automatically be signed on the master. +key will automatically be accepted by the master. Once a salt-minion has been successfully installed on the instance, connectivity to it can be verified with Salt: .. code-block:: bash - salt my-machine test.ping + salt my-machine test.version Destroy Options --------------- +.. versionadded:: Oxygen + For obvious reasons, the ``destroy`` action does not actually vaporize hardware. -If the salt master is connected using salt-api, it can tear down parts of -the client machines. It will remove the client's key from the salt master, -and will attempt the following options: +If the salt master is connected, it can tear down parts of the client machines. +It will remove the client's key from the salt master, +and can execute the following options: .. code-block:: yaml - remove_config_on_destroy: true # default: true # Deactivate salt-minion on reboot and - # delete the minion config and key files from its ``/etc/salt`` directory, - # NOTE: If deactivation is unsuccessful (older Ubuntu machines) then when + # delete the minion config and key files from its "/etc/salt" directory, + # NOTE: If deactivation was unsuccessful (older Ubuntu machines) then when # salt-minion restarts it will automatically create a new, unwanted, set - # of key files. The ``force_minion_config`` option must be used in that case. + # of key files. Use the "force_minion_config" option to replace them. - shutdown_on_destroy: false # default: false - # send a ``shutdown`` command to the client. - -.. versionadded:: Oxygen + # last of all, send a "shutdown" command to the client. Wake On LAN ----------- + +.. versionadded:: Oxygen + In addition to connecting a hardware machine to a Salt master, you have the option of sending a wake-on-LAN `magic packet`_ -to start the machine running. +to start that machine running. .. _magic packet: https://en.wikipedia.org/wiki/Wake-on-LAN From b9f7b41f95c9809962dd8b2aacc02a36c37a05d6 Mon Sep 17 00:00:00 2001 From: Sergey Kacheev Date: Fri, 20 Oct 2017 22:54:53 +0700 Subject: [PATCH 139/184] add ".. versionadded:: Oxygen" tag --- salt/modules/vault.py | 2 ++ salt/utils/vault.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/modules/vault.py b/salt/modules/vault.py index 8df9f2752a..6717e77f22 100644 --- a/salt/modules/vault.py +++ b/salt/modules/vault.py @@ -37,6 +37,8 @@ Functions to interact with Hashicorp Vault. For details please see http://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification + .. versionadded:: Oxygen + auth Currently only token auth is supported. The token must be able to create tokens with the policies that should be assigned to minions. Required. diff --git a/salt/utils/vault.py b/salt/utils/vault.py index fef7611209..b1f7eed8d7 100644 --- a/salt/utils/vault.py +++ b/salt/utils/vault.py @@ -126,8 +126,8 @@ def make_request(method, resource, profile=None, **args): connection = _get_vault_connection() token, vault_url = connection['token'], connection['url'] - if "verify" not in args: - args["verify"] = connection['verify'] + if 'verify' not in args: + args['verify'] = connection['verify'] url = "{0}/{1}".format(vault_url, resource) headers = {'X-Vault-Token': token, 'Content-Type': 'application/json'} From 61e2e9ccda075d1f0006765c1139852062e93dd7 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 20 Oct 2017 09:59:54 -0600 Subject: [PATCH 140/184] Fix some lint --- salt/utils/win_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/win_functions.py b/salt/utils/win_functions.py index efaac09f2e..9f9f1d7a10 100644 --- a/salt/utils/win_functions.py +++ b/salt/utils/win_functions.py @@ -144,7 +144,7 @@ def get_current_user(): def get_sam_name(username): - ''' + r''' Gets the SAM name for a user. It basically prefixes a username without a backslash with the computer name. If the user does not exist, a SAM compatible name will be returned using the local hostname as the domain. From f879f322a6260a90c422e26535246d40c507f458 Mon Sep 17 00:00:00 2001 From: Alex moore Date: Fri, 20 Oct 2017 14:04:20 -0600 Subject: [PATCH 141/184] Revert changes to pgjsonb from 553aaad105434d25c982a90b16b8387163d89ffa --- salt/returners/pgjsonb.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/salt/returners/pgjsonb.py b/salt/returners/pgjsonb.py index 30d8be02d6..dd09d31d78 100644 --- a/salt/returners/pgjsonb.py +++ b/salt/returners/pgjsonb.py @@ -395,30 +395,6 @@ def get_jids(): return ret -def get_jids_filter(count, filter_find_job=True): - ''' - Return a list of all job ids - :param int count: show not more than the count of most recent jobs - :param bool filter_find_jobs: filter out 'saltutil.find_job' jobs - ''' - with _get_serv(ret=None, commit=True) as cur: - - sql = '''SELECT * FROM ( - SELECT DISTINCT jid ,load FROM jids - {0} - ORDER BY jid DESC limit {1} - ) tmp - ORDER BY jid;''' - where = '''WHERE 'load' NOT LIKE '%"fun": "saltutil.find_job"%' ''' - - cur.execute(sql.format(where if filter_find_job else '', count)) - data = cur.fetchall() - ret = [] - for jid, load in data: - ret.append(salt.utils.jid.format_jid_instance_ext(jid, load)) - return ret - - def get_minions(): ''' Return a list of minions From 580feea5a7545a981131cd9a9626897b16f9b28f Mon Sep 17 00:00:00 2001 From: SaltyCharles Date: Fri, 20 Oct 2017 13:10:36 -0700 Subject: [PATCH 142/184] fix typos fix typo's for clarity --- salt/modules/boto_ec2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/boto_ec2.py b/salt/modules/boto_ec2.py index 376ec7fc0d..619c219e0d 100644 --- a/salt/modules/boto_ec2.py +++ b/salt/modules/boto_ec2.py @@ -154,7 +154,7 @@ def get_unassociated_eip_address(domain='standard', region=None, key=None, Return the first unassociated EIP domain - Indicates whether the address is a EC2 address or a VPC address + Indicates whether the address is an EC2 address or a VPC address (standard|vpc). CLI Example: @@ -771,9 +771,9 @@ def get_tags(instance_id=None, keyid=None, key=None, profile=None, def exists(instance_id=None, name=None, tags=None, region=None, key=None, keyid=None, profile=None, in_states=None, filters=None): ''' - Given a instance id, check to see if the given instance id exists. + Given an instance id, check to see if the given instance id exists. - Returns True if the given an instance with the given id, name, or tags + Returns True if the given instance with the given id, name, or tags exists; otherwise, False is returned. CLI Example: From 6d5409efb8e43f674c4994275b3ee06bd020ba75 Mon Sep 17 00:00:00 2001 From: SaltyCharles Date: Fri, 20 Oct 2017 13:13:16 -0700 Subject: [PATCH 143/184] fix typo fix typo for clarity --- salt/modules/boto_elasticache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/boto_elasticache.py b/salt/modules/boto_elasticache.py index 21de556b4c..d977e8b739 100644 --- a/salt/modules/boto_elasticache.py +++ b/salt/modules/boto_elasticache.py @@ -75,7 +75,7 @@ def __virtual__(): Only load if boto libraries exist. ''' if not HAS_BOTO: - return (False, 'The modle boto_elasticache could not be loaded: boto libraries not found') + return (False, 'The model boto_elasticache could not be loaded: boto libraries not found') __utils__['boto.assign_funcs'](__name__, 'elasticache', pack=__salt__) return True From 56e23f8773f1cdd70e059e640107e1d8695ad7dd Mon Sep 17 00:00:00 2001 From: SaltyCharles Date: Fri, 20 Oct 2017 13:18:50 -0700 Subject: [PATCH 144/184] fix typo fix typo for clarity --- salt/modules/boto_vpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/boto_vpc.py b/salt/modules/boto_vpc.py index 8c907d9479..babc6829a9 100644 --- a/salt/modules/boto_vpc.py +++ b/salt/modules/boto_vpc.py @@ -763,7 +763,7 @@ def describe_vpcs(vpc_id=None, name=None, cidr=None, tags=None, ''' Describe all VPCs, matching the filter criteria if provided. - Returns a a list of dictionaries with interesting properties. + Returns a list of dictionaries with interesting properties. .. versionadded:: 2015.8.0 From 3d068357c12815237fec8aeb41b0bb92e8b502bd Mon Sep 17 00:00:00 2001 From: SaltyCharles Date: Fri, 20 Oct 2017 13:25:14 -0700 Subject: [PATCH 145/184] fix typos fix typos for clarity --- salt/modules/cassandra_cql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/cassandra_cql.py b/salt/modules/cassandra_cql.py index b377a49556..afa2030d98 100644 --- a/salt/modules/cassandra_cql.py +++ b/salt/modules/cassandra_cql.py @@ -219,7 +219,7 @@ def _connect(contact_points=None, port=None, cql_user=None, cql_pass=None, # TODO: Call cluster.shutdown() when the module is unloaded on # master/minion shutdown. Currently, Master.shutdown() and Minion.shutdown() # do nothing to allow loaded modules to gracefully handle resources stored - # in __context__ (i.e. connection pools). This means that the the connection + # in __context__ (i.e. connection pools). This means that the connection # pool is orphaned and Salt relies on Cassandra to reclaim connections. # Perhaps if Master/Minion daemons could be enhanced to call an "__unload__" # function, or something similar for each loaded module, connection pools @@ -430,7 +430,7 @@ def cql_query_with_prepare(query, statement_name, statement_arguments, async=Fal values[key] = value ret.append(values) - # If this was a synchronous call, then we either have a empty list + # If this was a synchronous call, then we either have an empty list # because there was no return, or we have a return # If this was an async call we only return the empty list return ret From e33c94a89b1e76084e7f6e67eb2bc7a43cdc713f Mon Sep 17 00:00:00 2001 From: SaltyCharles Date: Fri, 20 Oct 2017 13:27:59 -0700 Subject: [PATCH 146/184] fix typos fix typos for clarity --- salt/modules/cmdmod.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index fbc8ad1fbf..c3c2070126 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -2773,8 +2773,8 @@ def shell_info(shell, list_modules=False): ''' regex_shells = { 'bash': [r'version (\d\S*)', 'bash', '--version'], - 'bash-test-error': [r'versioZ ([-\w.]+)', 'bash', '--version'], # used to test a error result - 'bash-test-env': [r'(HOME=.*)', 'bash', '-c', 'declare'], # used to test a error result + 'bash-test-error': [r'versioZ ([-\w.]+)', 'bash', '--version'], # used to test an error result + 'bash-test-env': [r'(HOME=.*)', 'bash', '-c', 'declare'], # used to test an error result 'zsh': [r'^zsh (\d\S*)', 'zsh', '--version'], 'tcsh': [r'^tcsh (\d\S*)', 'tcsh', '--version'], 'cmd': [r'Version ([\d.]+)', 'cmd.exe', '/C', 'ver'], From 730c1a390fdb653c4fe3fca3de0ef64f691b222a Mon Sep 17 00:00:00 2001 From: SaltyCharles Date: Fri, 20 Oct 2017 13:41:44 -0700 Subject: [PATCH 147/184] fix typo fix typo for clarity --- salt/modules/consul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/consul.py b/salt/modules/consul.py index 68a4bd3288..a6592cf1c8 100644 --- a/salt/modules/consul.py +++ b/salt/modules/consul.py @@ -1953,7 +1953,7 @@ def status_peers(consul_url): :param consul_url: The Consul server URL. :return: Retrieves the Raft peers for the - datacenter in which the the agent is running. + datacenter in which the agent is running. CLI Example: From 9b505d955be91a37457b07167f6c54e75614abec Mon Sep 17 00:00:00 2001 From: SaltyCharles Date: Fri, 20 Oct 2017 13:45:41 -0700 Subject: [PATCH 148/184] fix typos fix typos for clarity --- salt/modules/debbuild.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/debbuild.py b/salt/modules/debbuild.py index 37c7da1123..84069d2912 100644 --- a/salt/modules/debbuild.py +++ b/salt/modules/debbuild.py @@ -48,7 +48,7 @@ __virtualname__ = 'pkgbuild' def __virtual__(): ''' - Confirm this module is on a Debian based system, and has required utilities + Confirm this module is on a Debian-based system, and has required utilities ''' if __grains__.get('os_family', False) in ('Kali', 'Debian'): missing_util = False @@ -726,7 +726,7 @@ def make_repo(repodir, if times_looped > number_retries: raise SaltInvocationError( - 'Attemping to sign file {0} failed, timed out after {1} seconds' + 'Attempting to sign file {0} failed, timed out after {1} seconds' .format(abs_file, int(times_looped * interval)) ) time.sleep(interval) @@ -770,7 +770,7 @@ def make_repo(repodir, if times_looped > number_retries: raise SaltInvocationError( - 'Attemping to reprepro includedsc for file {0} failed, timed out after {1} loops'.format(abs_file, times_looped) + 'Attempting to reprepro includedsc for file {0} failed, timed out after {1} loops'.format(abs_file, times_looped) ) time.sleep(interval) From 7da57b877144ed3e1bd90d345be3f2301c74f655 Mon Sep 17 00:00:00 2001 From: SaltyCharles Date: Fri, 20 Oct 2017 13:49:19 -0700 Subject: [PATCH 149/184] fix typo fix typo/grammar for clarity --- salt/modules/debian_ip.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/modules/debian_ip.py b/salt/modules/debian_ip.py index 6b5c760c92..e68b771dd9 100644 --- a/salt/modules/debian_ip.py +++ b/salt/modules/debian_ip.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ''' -The networking module for Debian based distros +The networking module for Debian-based distros References: @@ -46,7 +46,7 @@ __virtualname__ = 'ip' def __virtual__(): ''' - Confine this module to Debian based distros + Confine this module to Debian-based distros ''' if __grains__['os_family'] == 'Debian': return __virtualname__ @@ -1562,7 +1562,7 @@ def _read_temp_ifaces(iface, data): return '' ifcfg = template.render({'name': iface, 'data': data}) - # Return as a array so the difflib works + # Return as an array so the difflib works return [item + '\n' for item in ifcfg.split('\n')] @@ -1616,7 +1616,7 @@ def _write_file_ifaces(iface, data, **settings): else: fout.write(ifcfg) - # Return as a array so the difflib works + # Return as an array so the difflib works return saved_ifcfg.split('\n') @@ -1646,7 +1646,7 @@ def _write_file_ppp_ifaces(iface, data): with salt.utils.files.fopen(filename, 'w') as fout: fout.write(ifcfg) - # Return as a array so the difflib works + # Return as an array so the difflib works return filename From 06047f6454c6fb7f64443417b478d3c7a7bbf5dc Mon Sep 17 00:00:00 2001 From: SaltyCharles Date: Fri, 20 Oct 2017 13:51:12 -0700 Subject: [PATCH 150/184] fix typo fix typo for clarity --- salt/modules/dockercompose.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/dockercompose.py b/salt/modules/dockercompose.py index a73321e9cf..345f572359 100644 --- a/salt/modules/dockercompose.py +++ b/salt/modules/dockercompose.py @@ -686,7 +686,7 @@ def ps(path): def up(path, service_names=None): ''' - Create and start containers defined in the the docker-compose.yml file + Create and start containers defined in the docker-compose.yml file located in path, service_names is a python list, if omitted create and start all containers From a2a1a4be9c4b04c3c980de373a3a0e5786dc5e08 Mon Sep 17 00:00:00 2001 From: SaltyCharles Date: Fri, 20 Oct 2017 13:53:28 -0700 Subject: [PATCH 151/184] fix typo fix typo for clarity --- salt/modules/drac.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/drac.py b/salt/modules/drac.py index 51d44cab75..24919c75c1 100644 --- a/salt/modules/drac.py +++ b/salt/modules/drac.py @@ -465,5 +465,5 @@ def server_pxe(): log.warning('failed to set boot order') return False - log.warning('failed to to configure PXE boot') + log.warning('failed to configure PXE boot') return False From 0ed4f5533d85aa38fad6054bac80c241e48368b6 Mon Sep 17 00:00:00 2001 From: SaltyCharles Date: Fri, 20 Oct 2017 13:54:40 -0700 Subject: [PATCH 152/184] fix typo fix typo for clarity --- salt/modules/dracr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/dracr.py b/salt/modules/dracr.py index d42ae4b83d..38f722a423 100644 --- a/salt/modules/dracr.py +++ b/salt/modules/dracr.py @@ -923,7 +923,7 @@ def server_pxe(host=None, log.warning('failed to set boot order') return False - log.warning('failed to to configure PXE boot') + log.warning('failed to configure PXE boot') return False From 3b46743207c95469564371bba03bed021f58ef4d Mon Sep 17 00:00:00 2001 From: SaltyCharles Date: Fri, 20 Oct 2017 13:56:04 -0700 Subject: [PATCH 153/184] fix typo fix typo for clarity --- salt/modules/environ.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/environ.py b/salt/modules/environ.py index 26972f4ca9..e06a8aecaa 100644 --- a/salt/modules/environ.py +++ b/salt/modules/environ.py @@ -45,7 +45,7 @@ def setval(key, val, false_unsets=False, permanent=False): permanent On Windows minions this will set the environment variable in the - registry so that it is always added as a environment variable when + registry so that it is always added as an environment variable when applications open. If you want to set the variable to HKLM instead of HKCU just pass in "HKLM" for this parameter. On all other minion types this will be ignored. Note: This will only take affect on applications @@ -144,7 +144,7 @@ def setenv(environ, false_unsets=False, clear_all=False, update_minion=False, pe permanent On Windows minions this will set the environment variable in the - registry so that it is always added as a environment variable when + registry so that it is always added as an environment variable when applications open. If you want to set the variable to HKLM instead of HKCU just pass in "HKLM" for this parameter. On all other minion types this will be ignored. Note: This will only take affect on applications From 1f91ee9d78e7f6b914b8980736241284adb0c859 Mon Sep 17 00:00:00 2001 From: SaltyCharles Date: Fri, 20 Oct 2017 13:57:58 -0700 Subject: [PATCH 154/184] fix typo fix typo for clarity --- salt/modules/freebsdports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/freebsdports.py b/salt/modules/freebsdports.py index daa3b6fd9a..5779a8a8dc 100644 --- a/salt/modules/freebsdports.py +++ b/salt/modules/freebsdports.py @@ -6,7 +6,7 @@ Install software from the FreeBSD ``ports(7)`` system This module allows you to install ports using ``BATCH=yes`` to bypass configuration prompts. It is recommended to use the :mod:`ports state -` to install ports, but it it also possible to use +` to install ports, but it is also possible to use this module exclusively from the command line. .. code-block:: bash From 74622436e4165f5c6af166d787d1e5d2840e1cd1 Mon Sep 17 00:00:00 2001 From: SaltyCharles Date: Fri, 20 Oct 2017 14:01:05 -0700 Subject: [PATCH 155/184] fix typos fix typos for clarity --- salt/modules/genesis.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/modules/genesis.py b/salt/modules/genesis.py index ab0eed5138..9af3e2cbc5 100644 --- a/salt/modules/genesis.py +++ b/salt/modules/genesis.py @@ -306,7 +306,7 @@ def _bootstrap_yum( root The root of the image to install to. Will be created as a directory if - if does not exist. (e.x.: /root/arch) + it does not exist. (e.x.: /root/arch) pkg_confs The location of the conf files to copy into the image, to point yum @@ -374,7 +374,7 @@ def _bootstrap_deb( root The root of the image to install to. Will be created as a directory if - if does not exist. (e.x.: /root/wheezy) + it does not exist. (e.x.: /root/wheezy) arch Architecture of the target image. (e.x.: amd64) @@ -472,7 +472,7 @@ def _bootstrap_pacman( root The root of the image to install to. Will be created as a directory if - if does not exist. (e.x.: /root/arch) + it does not exist. (e.x.: /root/arch) pkg_confs The location of the conf files to copy into the image, to point pacman @@ -480,7 +480,7 @@ def _bootstrap_pacman( img_format The image format to be used. The ``dir`` type needs no special - treatment, but others need special treatement. + treatment, but others need special treatment. pkgs A list of packages to be installed on this image. For Arch Linux, this From 10d13062ec48acfc46dba1ea1301f8ca340cd1c7 Mon Sep 17 00:00:00 2001 From: Eric Radman Date: Sat, 21 Oct 2017 00:43:45 -0400 Subject: [PATCH 156/184] Mock open to /var/log/btmp and /var/log/wtmp - btmp may not exist - wtmp may not exist, or it may not contain ASCII data Unbreaks tests on OpenBSD 6.2 --- tests/unit/beacons/test_btmp_beacon.py | 6 ++++-- tests/unit/beacons/test_wtmp_beacon.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/unit/beacons/test_btmp_beacon.py b/tests/unit/beacons/test_btmp_beacon.py index 708dae9454..d4bf94abf4 100644 --- a/tests/unit/beacons/test_btmp_beacon.py +++ b/tests/unit/beacons/test_btmp_beacon.py @@ -59,8 +59,10 @@ class BTMPBeaconTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(ret, (True, 'Valid beacon configuration')) - ret = btmp.beacon(config) - self.assertEqual(ret, []) + with patch('salt.utils.files.fopen', mock_open()) as m_open: + ret = btmp.beacon(config) + m_open.assert_called_with(btmp.BTMP, 'rb') + self.assertEqual(ret, []) def test_match(self): with patch('salt.utils.files.fopen', diff --git a/tests/unit/beacons/test_wtmp_beacon.py b/tests/unit/beacons/test_wtmp_beacon.py index b1edd97096..c28f3554b6 100644 --- a/tests/unit/beacons/test_wtmp_beacon.py +++ b/tests/unit/beacons/test_wtmp_beacon.py @@ -60,8 +60,10 @@ class WTMPBeaconTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(ret, (True, 'Valid beacon configuration')) - ret = wtmp.beacon(config) - self.assertEqual(ret, []) + with patch('salt.utils.files.fopen', mock_open()) as m_open: + ret = wtmp.beacon(config) + m_open.assert_called_with(wtmp.WTMP, 'rb') + self.assertEqual(ret, []) def test_match(self): with patch('salt.utils.files.fopen', From f19e50a35199858059cf0901bae3919f61eb61b5 Mon Sep 17 00:00:00 2001 From: me Date: Sun, 22 Oct 2017 11:02:04 +0330 Subject: [PATCH 157/184] change service to process in module docstring --- salt/beacons/ps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/beacons/ps.py b/salt/beacons/ps.py index b1f18431f4..d65be4c12e 100644 --- a/salt/beacons/ps.py +++ b/salt/beacons/ps.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ''' -Send events covering service status +Send events covering process status ''' # Import Python Libs From 5dd9fe89184071a54734184aa2c3aa728d57b9f0 Mon Sep 17 00:00:00 2001 From: Xavier G Date: Sun, 22 Oct 2017 16:21:19 +0200 Subject: [PATCH 158/184] grafana4: dashboard: do not override the title. grafana4_dashboard.present() was liable to override the user-provided dashboard title with its name (which is actually the slug, as hinted in the documentation of grafana4.get_dashboard()). It makes sense for grafana4_dashboard.present() to enforce a title to fulfill Grafana's requirements, but the slug (normalized, non-numerical, URL-friendly identifier) should be used as a title (human-friendly identifier) only as a last resort. --- salt/states/grafana4_dashboard.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/states/grafana4_dashboard.py b/salt/states/grafana4_dashboard.py index 0f3318c3b8..737873afcf 100644 --- a/salt/states/grafana4_dashboard.py +++ b/salt/states/grafana4_dashboard.py @@ -108,7 +108,8 @@ def present(name, # Build out all dashboard fields new_dashboard = _inherited_dashboard( dashboard, base_dashboards_from_pillar, ret) - new_dashboard['title'] = name + if 'title' not in new_dashboard: + new_dashboard['title'] = name rows = new_dashboard.get('rows', []) for i, row in enumerate(rows): rows[i] = _inherited_row(row, base_rows_from_pillar, ret) From c0a5880d329202990089fec91667dd2fd28cadb6 Mon Sep 17 00:00:00 2001 From: Xavier G Date: Sun, 22 Oct 2017 16:36:10 +0200 Subject: [PATCH 159/184] grafana4: datasource: handle missing keys in datasource description. --- salt/states/grafana4_datasource.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/states/grafana4_datasource.py b/salt/states/grafana4_datasource.py index 7ae3ef3e95..d4b698daf9 100644 --- a/salt/states/grafana4_datasource.py +++ b/salt/states/grafana4_datasource.py @@ -151,6 +151,12 @@ def present(name, ret['changes'] = data return ret + # At this stage, the datasource exists; however, the object provided by + # Grafana may lack some null keys compared to our "data" dict: + for key in data: + if key not in datasource: + datasource[key] = None + if data == datasource: ret['changes'] = None ret['comment'] = 'Data source {0} already up-to-date'.format(name) From a1f27c9f00d9ad43595a8c4f5f583f99c2a8ae1f Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Mon, 23 Oct 2017 13:17:49 +0000 Subject: [PATCH 160/184] Add explicit non-zero retcode to napalm config functions Thanks to @dmitrykuzmenko's explanation in #43187, we can add explicit return code for napalm configuration related functions: when loading configuration fails for a reason or another (there can be several different reasons), the function will be considered as "failed". This will not change any behaviour on the CLI, but it is useful in the --summary mode to see what minions failed. --- salt/modules/napalm_network.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/modules/napalm_network.py b/salt/modules/napalm_network.py index 5042eee025..341df38cc9 100644 --- a/salt/modules/napalm_network.py +++ b/salt/modules/napalm_network.py @@ -153,6 +153,7 @@ def _config_logic(napalm_device, loaded_result['diff'] = None loaded_result['result'] = False loaded_result['comment'] = _compare.get('comment') + __context__['retcode'] = 1 return loaded_result _loaded_res = loaded_result.get('result', False) @@ -172,12 +173,15 @@ def _config_logic(napalm_device, # make sure it notifies # that something went wrong _explicit_close(napalm_device) + __context__['retcode'] = 1 return loaded_result loaded_result['comment'] += 'Configuration discarded.' # loaded_result['result'] = False not necessary # as the result can be true when test=True _explicit_close(napalm_device) + if not loaded_result['result']: + __context__['retcode'] = 1 return loaded_result if not test and commit_config: @@ -208,10 +212,13 @@ def _config_logic(napalm_device, loaded_result['result'] = False # notify if anything goes wrong _explicit_close(napalm_device) + __context__['retcode'] = 1 return loaded_result loaded_result['already_configured'] = True loaded_result['comment'] = 'Already configured.' _explicit_close(napalm_device) + if not loaded_result['result']: + __context__['retcode'] = 1 return loaded_result From 3a10b6aef12b3a79dc623dde6d5f70760049931b Mon Sep 17 00:00:00 2001 From: Ric Klaren Date: Sat, 21 Oct 2017 15:24:27 -0500 Subject: [PATCH 161/184] Fixes #44227, make salt-cloud/libvirt cleanup after errors more robust Fix vm cleanup when salt bootstrap fails when using salt-cloud with the libvirt provider. Signed-off-by: Ric Klaren --- salt/cloud/clouds/libvirt.py | 39 +++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/salt/cloud/clouds/libvirt.py b/salt/cloud/clouds/libvirt.py index 1da5925f8f..4554464164 100644 --- a/salt/cloud/clouds/libvirt.py +++ b/salt/cloud/clouds/libvirt.py @@ -462,18 +462,39 @@ def create(vm_): return ret except Exception as e: # pylint: disable=broad-except - # Try to clean up in as much cases as possible - log.info('Cleaning up after exception clean up items: {0}'.format(cleanup)) - for leftover in cleanup: - what = leftover['what'] - item = leftover['item'] - if what == 'domain': - destroy_domain(conn, item) - if what == 'volume': - item.delete() + do_cleanup(cleanup) raise e +def do_cleanup(cleanup): + # Try to clean up in as many cases as possible. Libvirt behavior changed + # somewhat over time, so try to be robust. + log.info('Cleaning up after exception') + for leftover in cleanup: + what = leftover['what'] + item = leftover['item'] + if what == 'domain': + log.info('Cleaning up {0} {1}'.format(what, item.name())) + try: + item.destroy() + log.debug('{0} {1} forced off'.format(what, item.name())) + except libvirtError: + pass + try: + item.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_MANAGED_SAVE+ + libvirt.VIR_DOMAIN_UNDEFINE_SNAPSHOTS_METADATA+ + libvirt.VIR_DOMAIN_UNDEFINE_NVRAM) + log.debug('{0} {1} undefined'.format(what, item.name())) + except libvirtError: + pass + if what == 'volume': + try: + item.delete() + log.debug('{0} {1} cleaned up'.format(what, item.name())) + except libvirtError: + pass + + def destroy(name, call=None): """ This function irreversibly destroys a virtual machine on the cloud provider. From 7917d1e61e268414ae861325dc2ec900e36daca4 Mon Sep 17 00:00:00 2001 From: Ric Klaren Date: Mon, 23 Oct 2017 09:26:37 -0500 Subject: [PATCH 162/184] Incorporate review comments. --- salt/cloud/clouds/libvirt.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/salt/cloud/clouds/libvirt.py b/salt/cloud/clouds/libvirt.py index 4554464164..f88ae566dd 100644 --- a/salt/cloud/clouds/libvirt.py +++ b/salt/cloud/clouds/libvirt.py @@ -463,12 +463,27 @@ def create(vm_): return ret except Exception as e: # pylint: disable=broad-except do_cleanup(cleanup) + # throw the root cause after cleanup raise e def do_cleanup(cleanup): - # Try to clean up in as many cases as possible. Libvirt behavior changed - # somewhat over time, so try to be robust. + ''' + Clean up clone domain leftovers as much as possible. + + Extra robust clean up in order to deal with some small changes in libvirt + behavior over time. Passed in volumes and domains are deleted, any errors + are ignored. Used when cloning/provisioning a domain fails. + + :param cleanup: list containing dictonaries with two keys: 'what' and 'item'. + If 'what' is domain the 'item' is a libvirt domain object. + If 'what' is volume then the item is a libvirt volume object. + + Returns: + none + + .. versionadded: 2017.7.3 + ''' log.info('Cleaning up after exception') for leftover in cleanup: what = leftover['what'] From 9eacaeb6af45d2ce9a6a8508e2fb58fee687bc2d Mon Sep 17 00:00:00 2001 From: Bike Dude Date: Mon, 23 Oct 2017 21:35:32 +0200 Subject: [PATCH 163/184] small docs fix in system.py --- salt/modules/system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/system.py b/salt/modules/system.py index 87673a372e..650bbeba51 100644 --- a/salt/modules/system.py +++ b/salt/modules/system.py @@ -596,7 +596,7 @@ def set_computer_name(hostname): .. code-block:: bash - salt '*' system.set_conputer_name master.saltstack.com + salt '*' system.set_computer_name master.saltstack.com ''' return __salt__['network.mod_hostname'](hostname) From 4729ccd32b958c8dbc369ba3f4413ffa07ffae13 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 23 Oct 2017 17:13:32 -0400 Subject: [PATCH 164/184] Add multiple spm integration tests --- tests/integration/spm/test_files.py | 36 +++++++++++++ tests/integration/spm/test_info.py | 36 +++++++++++++ tests/integration/spm/test_install.py | 50 +++++++++++++++++ tests/integration/spm/test_repo.py | 36 +++++++++++++ tests/support/case.py | 78 +++++++++++++++++++++++++-- 5 files changed, 232 insertions(+), 4 deletions(-) create mode 100644 tests/integration/spm/test_files.py create mode 100644 tests/integration/spm/test_info.py create mode 100644 tests/integration/spm/test_install.py create mode 100644 tests/integration/spm/test_repo.py diff --git a/tests/integration/spm/test_files.py b/tests/integration/spm/test_files.py new file mode 100644 index 0000000000..7b8ae7f5db --- /dev/null +++ b/tests/integration/spm/test_files.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +''' +Tests for the spm files utility +''' +# Import python libs +from __future__ import absolute_import +import os +import shutil + +# Import Salt Testing libs +from tests.support.case import SPMCase +from tests.support.helpers import destructiveTest + + +@destructiveTest +class SPMFilesTest(SPMCase): + ''' + Validate the spm files command + ''' + def setUp(self): + self.config = self._spm_config() + self._spm_build_files(self.config) + + def test_spm_files(self): + ''' + test spm files + ''' + self._spm_create_update_repo(self.config) + install = self.run_spm('install', self.config, 'apache') + get_files = self.run_spm('files', self.config, 'apache') + + os.path.exists(os.path.join(self.config['formula_path'], 'apache', + 'apache.sls')) + + def tearDown(self): + shutil.rmtree(self._tmp_spm) diff --git a/tests/integration/spm/test_info.py b/tests/integration/spm/test_info.py new file mode 100644 index 0000000000..d6ab4dcf3b --- /dev/null +++ b/tests/integration/spm/test_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +''' +Tests for the spm info utility +''' +# Import python libs +from __future__ import absolute_import +import shutil + +# Import Salt Testing libs +from tests.support.case import SPMCase +from tests.support.helpers import destructiveTest + + +@destructiveTest +class SPMInfoTest(SPMCase): + ''' + Validate the spm info command + ''' + def setUp(self): + self.config = self._spm_config() + self._spm_build_files(self.config) + + def test_spm_info(self): + ''' + test spm build + ''' + self._spm_create_update_repo(self.config) + install = self.run_spm('install', self.config, 'apache') + get_info = self.run_spm('info', self.config, 'apache') + + check_info = ['Supported OSes', 'Supported OS', 'installing Apache'] + for info in check_info: + self.assertIn(info, ''.join(get_info)) + + def tearDown(self): + shutil.rmtree(self._tmp_spm) diff --git a/tests/integration/spm/test_install.py b/tests/integration/spm/test_install.py new file mode 100644 index 0000000000..30bca50b96 --- /dev/null +++ b/tests/integration/spm/test_install.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +''' +Tests for the spm install utility +''' +# Import python libs +from __future__ import absolute_import +import os +import shutil + +# Import Salt Testing libs +from tests.support.case import SPMCase +from tests.support.helpers import destructiveTest + + +@destructiveTest +class SPMInstallTest(SPMCase): + ''' + Validate the spm install command + ''' + def setUp(self): + self.config = self._spm_config() + self._spm_build_files(self.config) + + def test_spm_install_local_dir(self): + ''' + test spm install from local directory + ''' + build_spm = self.run_spm('build', self.config, self.formula_dir) + spm_file = os.path.join(self.config['spm_build_dir'], + 'apache-201506-2.spm') + + install = self.run_spm('install', self.config, spm_file) + + sls = os.path.join(self.config['formula_path'], 'apache', 'apache.sls') + + self.assertTrue(os.path.exists(sls)) + + def test_spm_install_from_repo(self): + ''' + test spm install from repo + ''' + self._spm_create_update_repo(self.config) + install = self.run_spm('install', self.config, 'apache') + + sls = os.path.join(self.config['formula_path'], 'apache', 'apache.sls') + + self.assertTrue(os.path.exists(sls)) + + def tearDown(self): + shutil.rmtree(self._tmp_spm) diff --git a/tests/integration/spm/test_repo.py b/tests/integration/spm/test_repo.py new file mode 100644 index 0000000000..a4daf10a63 --- /dev/null +++ b/tests/integration/spm/test_repo.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +''' +Tests for the spm repo +''' +# Import python libs +from __future__ import absolute_import +import os +import shutil + +# Import Salt Testing libs +from tests.support.case import SPMCase +from tests.support.helpers import destructiveTest + + +@destructiveTest +class SPMRepoTest(SPMCase): + ''' + Validate commands related to spm repo + ''' + def setUp(self): + self.config = self._spm_config() + self._spm_build_files(self.config) + + def test_spm_create_update_repo(self): + ''' + test spm create_repo + ''' + self._spm_create_update_repo(self.config) + + self.assertTrue(os.path.exists(self.config['spm_db'])) + + l_repo_file = os.path.join(self.config['spm_cache_dir'], 'local_repo.p') + self.assertTrue(os.path.exists(l_repo_file)) + + def tearDown(self): + shutil.rmtree(self._tmp_spm) diff --git a/tests/support/case.py b/tests/support/case.py index b620c69c72..91cb2663ad 100644 --- a/tests/support/case.py +++ b/tests/support/case.py @@ -22,6 +22,7 @@ import time import stat import errno import signal +import textwrap import logging import tempfile import subprocess @@ -564,11 +565,60 @@ class ShellCase(ShellTestCase, AdaptedConfigurationTestCaseMixin, ScriptPathMixi timeout=timeout) +class SPMTestUserInterface(object): + ''' + Test user interface to SPMClient + ''' + def __init__(self): + self._status = [] + self._confirm = [] + self._error = [] + + def status(self, msg): + self._status.append(msg) + + def confirm(self, action): + self._confirm.append(action) + + def error(self, msg): + self._error.append(msg) + + class SPMCase(TestCase, AdaptedConfigurationTestCaseMixin): ''' Class for handling spm commands ''' + def _spm_build_files(self, config): + self.formula_dir = os.path.join(' '.join(config['file_roots']['base']), 'formulas') + self.formula_sls_dir = os.path.join(self.formula_dir, 'apache') + self.formula_sls = os.path.join(self.formula_sls_dir, 'apache.sls') + self.formula_file = os.path.join(self.formula_dir, 'FORMULA') + + dirs = [self.formula_dir, self.formula_sls_dir] + for f_dir in dirs: + os.makedirs(f_dir) + + import salt.utils + + with salt.utils.fopen(self.formula_sls, 'w') as fp: + fp.write(textwrap.dedent('''\ + install-apache: + pkg.installed: + - name: apache2 + ''')) + + with salt.utils.fopen(self.formula_file, 'w') as fp: + fp.write(textwrap.dedent('''\ + name: apache + os: RedHat, Debian, Ubuntu, Suse, FreeBSD + os_family: RedHat, Debian, Suse, FreeBSD + version: 201506 + release: 2 + summary: Formula for installing Apache + description: Formula for installing Apache + ''')) + def _spm_config(self): self._tmp_spm = tempfile.mkdtemp() config = self.get_temp_config('minion', **{ @@ -595,16 +645,36 @@ class SPMCase(TestCase, AdaptedConfigurationTestCaseMixin): }) return config + def _spm_create_update_repo(self, config): + + build_spm = self.run_spm('build', self.config, self.formula_dir) + + c_repo = self.run_spm('create_repo', self.config, + self.config['spm_build_dir']) + + repo_conf_dir = self.config['spm_repos_config'] + '.d' + os.makedirs(repo_conf_dir) + + import salt.utils + + with salt.utils.fopen(os.path.join(repo_conf_dir, 'spm.repo'), 'w') as fp: + fp.write(textwrap.dedent('''\ + local_repo: + url: file://{0} + '''.format(self.config['spm_build_dir']))) + + u_repo = self.run_spm('update_repo', self.config) + def _spm_client(self, config): import salt.spm - ui = salt.spm.SPMCmdlineInterface() - client = salt.spm.SPMClient(ui, config) + self.ui = SPMTestUserInterface() + client = salt.spm.SPMClient(self.ui, config) return client - def run_spm(self, cmd, config, arg=()): + def run_spm(self, cmd, config, arg=None): client = self._spm_client(config) spm_cmd = client.run([cmd, arg]) - return spm_cmd + return self.ui._status class ModuleCase(TestCase, SaltClientTestCaseMixin): From 9e2e785034ab2e6b6077c3ff7f3015f7bd6bc53c Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Tue, 24 Oct 2017 10:33:53 -0400 Subject: [PATCH 165/184] add spm tests to test runner --- tests/runtests.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/runtests.py b/tests/runtests.py index 21c457a9b3..a3a0dc5ccd 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -130,6 +130,9 @@ TEST_SUITES = { 'returners': {'display_name': 'Returners', 'path': 'integration/returners'}, + 'spm': + {'display_name': 'SPM', + 'path': 'integration/spm'}, 'loader': {'display_name': 'Loader', 'path': 'integration/loader'}, @@ -338,6 +341,13 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): action='store_true', help='Run salt/returners/*.py tests' ) + self.test_selection_group.add_option( + '--spm', + dest='spm', + default=False, + action='store_true', + help='Run spm integration tests' + ) self.test_selection_group.add_option( '-l', '--loader', From 6ab82394bed804986373b04f0f5d7d228719f018 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 20 Oct 2017 16:57:37 -0600 Subject: [PATCH 166/184] Set up mocking Move the code that sets up the group object, computer object and gets existing members out of the individual functions so they can be mocked Mock the GroupObj Mock the ComputerObj Mock existing members Still need to fix info, getent, and members --- salt/modules/win_groupadd.py | 61 +++---- tests/unit/modules/test_win_groupadd.py | 219 +++++++++++++++++++++--- 2 files changed, 223 insertions(+), 57 deletions(-) diff --git a/salt/modules/win_groupadd.py b/salt/modules/win_groupadd.py index 17d0ffdffb..322f172fd2 100644 --- a/salt/modules/win_groupadd.py +++ b/salt/modules/win_groupadd.py @@ -36,6 +36,12 @@ def __virtual__(): return (False, "Module win_groupadd: module only works on Windows systems") +def _get_computer_object(): + pythoncom.CoInitialize() + nt = win32com.client.Dispatch('AdsNameSpaces') + return nt.GetObject('', 'WinNT://.,computer') + + def add(name, **kwargs): ''' Add the specified group @@ -60,10 +66,8 @@ def add(name, **kwargs): 'comment': ''} if not info(name): - pythoncom.CoInitialize() - nt = win32com.client.Dispatch('AdsNameSpaces') + compObj = _get_computer_object() try: - compObj = nt.GetObject('', 'WinNT://.,computer') newGroup = compObj.Create('group', name) newGroup.SetInfo() ret['changes'].append('Successfully created group {0}'.format(name)) @@ -104,10 +108,8 @@ def delete(name, **kwargs): 'comment': ''} if info(name): - pythoncom.CoInitialize() - nt = win32com.client.Dispatch('AdsNameSpaces') + compObj = _get_computer_object() try: - compObj = nt.GetObject('', 'WinNT://.,computer') compObj.Delete('group', name) ret['changes'].append(('Successfully removed group {0}').format(name)) except pywintypes.com_error as com_err: @@ -144,14 +146,12 @@ def info(name): salt '*' group.info foo ''' - pythoncom.CoInitialize() - nt = win32com.client.Dispatch('AdsNameSpaces') - + groupObj = _get_group_object(name) + existing_members = _get_group_members(groupObj) try: - groupObj = nt.GetObject('', 'WinNT://./' + name + ',group') gr_name = groupObj.Name gr_mem = [] - for member in groupObj.members(): + for member in existing_members: gr_mem.append( member.ADSPath.replace('WinNT://', '').replace( '/', '\\').encode('ascii', 'backslashreplace')) @@ -213,6 +213,21 @@ def getent(refresh=False): return ret +def _get_group_object(name): + pythoncom.CoInitialize() + nt = win32com.client.Dispatch('AdsNameSpaces') + return nt.GetObject('', 'WinNT://./' + name + ',group') + + +def _get_group_members(groupObj): + existingMembers = [] + for member in groupObj.members(): + existingMembers.append( + member.ADSPath.replace('WinNT://', '').replace( + '/', '\\').encode('ascii', 'backslashreplace').lower()) + return existingMembers + + def adduser(name, username, **kwargs): ''' Add a user to a group @@ -240,17 +255,11 @@ def adduser(name, username, **kwargs): 'changes': {'Users Added': []}, 'comment': ''} - pythoncom.CoInitialize() - nt = win32com.client.Dispatch('AdsNameSpaces') - groupObj = nt.GetObject('', 'WinNT://./' + name + ',group') - existingMembers = [] - for member in groupObj.members(): - existingMembers.append( - member.ADSPath.replace('WinNT://', '').replace( - '/', '\\').encode('ascii', 'backslashreplace').lower()) + groupObj = _get_group_object(name) + existingMembers = _get_group_members(groupObj) try: - if salt.utils.win_functions.get_sam_name(username).lower() not in existingMembers: + if salt.utils.win_functions.get_sam_name(username) not in existingMembers: if not __opts__['test']: groupObj.Add('WinNT://' + username.replace('\\', '/')) @@ -299,17 +308,11 @@ def deluser(name, username, **kwargs): 'changes': {'Users Removed': []}, 'comment': ''} - pythoncom.CoInitialize() - nt = win32com.client.Dispatch('AdsNameSpaces') - groupObj = nt.GetObject('', 'WinNT://./' + name + ',group') - existingMembers = [] - for member in groupObj.members(): - existingMembers.append( - member.ADSPath.replace('WinNT://', '').replace( - '/', '\\').encode('ascii', 'backslashreplace').lower()) + groupObj = _get_group_object(name) + existingMembers = _get_group_members(groupObj) try: - if salt.utils.win_functions.get_sam_name(username).lower() in existingMembers: + if salt.utils.win_functions.get_sam_name(username) in existingMembers: if not __opts__['test']: groupObj.Remove('WinNT://' + username.replace('\\', '/')) diff --git a/tests/unit/modules/test_win_groupadd.py b/tests/unit/modules/test_win_groupadd.py index 9ddd7430ee..89426b21a0 100644 --- a/tests/unit/modules/test_win_groupadd.py +++ b/tests/unit/modules/test_win_groupadd.py @@ -10,6 +10,8 @@ from __future__ import absolute_import from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import TestCase, skipIf from tests.support.mock import ( + MagicMock, + Mock, patch, NO_MOCK, NO_MOCK_REASON @@ -45,21 +47,103 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): def test_add(self): ''' - Test if it add the specified group + Test adding a new group ''' - self.assertDictEqual(win_groupadd.add('foo'), - {'changes': [], 'name': 'foo', 'result': None, - 'comment': 'The group foo already exists.'}) + info = MagicMock(return_value=False) + with patch.object(win_groupadd, 'info', info),\ + patch.object(win_groupadd, '_get_computer_object', Mock()): + self.assertDictEqual(win_groupadd.add('foo'), + {'changes': ['Successfully created group foo'], + 'name': 'foo', + 'result': True, + 'comment': ''}) + + def test_add_group_exists(self): + ''' + Test adding a new group if the group already exists + ''' + info = MagicMock(return_value={'name': 'foo', + 'passwd': None, + 'gid': None, + 'members': ['HOST\\spongebob']}) + with patch.object(win_groupadd, 'info', info),\ + patch.object(win_groupadd, '_get_computer_object', Mock()): + self.assertDictEqual(win_groupadd.add('foo'), + {'changes': [], 'name': 'foo', 'result': None, + 'comment': 'The group foo already exists.'}) + + def test_add_error(self): + ''' + Test adding a group and encountering an error + ''' + class CompObj(object): + def Create(self, type, name): + raise pywintypes.com_error(-2147352567, 'Exception occurred.', (0, None, 'C', None, 0, -2146788248), None) + + compobj_mock = MagicMock(return_value=CompObj()) + + info = MagicMock(return_value=False) + with patch.object(win_groupadd, 'info', info),\ + patch.object(win_groupadd, '_get_computer_object', compobj_mock): + self.assertDictEqual(win_groupadd.add('foo'), + {'changes': [], + 'name': 'foo', + 'result': False, + 'comment': 'Failed to create group foo. C'}) # 'delete' function tests: 1 def test_delete(self): ''' - Test if it remove the specified group + Test removing a group ''' - self.assertDictEqual(win_groupadd.delete('foo'), - {'changes': [], 'name': 'foo', 'result': None, - 'comment': 'The group foo does not exists.'}) + info = MagicMock(return_value={'name': 'foo', + 'passwd': None, + 'gid': None, + 'members': ['HOST\\spongebob']}) + with patch.object(win_groupadd, 'info', info), \ + patch.object(win_groupadd, '_get_computer_object', Mock()): + self.assertDictEqual( + win_groupadd.delete('foo'), + {'changes': ['Successfully removed group foo'], + 'name': 'foo', + 'result': True, + 'comment': ''}) + + def test_delete_no_group(self): + ''' + Test removing a group that doesn't exists + ''' + info = MagicMock(return_value=False) + with patch.object(win_groupadd, 'info', info), \ + patch.object(win_groupadd, '_get_computer_object', Mock()): + self.assertDictEqual(win_groupadd.delete('foo'), + {'changes': [], 'name': 'foo', 'result': None, + 'comment': 'The group foo does not exists.'}) + + def test_delete_error(self): + ''' + Test removing a group and encountering an error + ''' + class CompObj(object): + def Delete(self, type, name): + raise pywintypes.com_error(-2147352567, 'Exception occurred.', (0, None, 'C', None, 0, -2146788248), None) + + compobj_mock = MagicMock(return_value=CompObj()) + + info = MagicMock(return_value={'name': 'foo', + 'passwd': None, + 'gid': None, + 'members': ['HOST\\spongebob']}) + with patch.object(win_groupadd, 'info', info),\ + patch.object(win_groupadd, '_get_computer_object', compobj_mock): + self.assertDictEqual( + win_groupadd.delete('foo'), + {'changes': [], + 'name': 'foo', + 'result': False, + 'comment': 'Failed to remove group foo. C'}) + # 'info' function tests: 1 @@ -67,12 +151,14 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): ''' Test if it return information about a group. ''' - with patch(win_groupadd.win32.client, 'flag', None): - self.assertDictEqual(win_groupadd.info('dc=salt'), + members = MagicMock(return_value=['HOST\\steve']) + with patch.object(win_groupadd, '_get_group_object', Mock()), \ + patch.object(win_groupadd, '_get_group_members', members): + self.assertDictEqual(win_groupadd.info('salt'), {'gid': None, - 'members': ['dc=\\user1'], + 'members': ['user1'], 'passwd': None, - 'name': 'WinNT://./dc=salt,group'}) + 'name': 'salt'}) with patch(win_groupadd.win32.client, 'flag', 1): self.assertFalse(win_groupadd.info('dc=salt')) @@ -93,31 +179,108 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): def test_adduser(self): ''' - Test if it add a user to a group + Test adding a user to a group ''' - with patch(win_groupadd.win32.client, 'flag', None): - self.assertDictEqual(win_groupadd.adduser('dc=foo', 'dc=\\username'), - {'changes': {'Users Added': ['dc=\\username']}, - 'comment': '', 'name': 'dc=foo', 'result': True}) + members = MagicMock(return_value=['HOST\\steve']) + with patch.object(win_groupadd, '_get_group_object', Mock()),\ + patch.object(win_groupadd, '_get_group_members', members),\ + patch('salt.utils.win_functions.get_sam_name', return_value='HOST\\spongebob'): + self.assertDictEqual( + win_groupadd.adduser('foo', 'spongebob'), + {'changes': {'Users Added': ['spongebob']}, + 'comment': '', + 'name': 'foo', + 'result': True}) - with patch(win_groupadd.win32.client, 'flag', 1): - comt = ('Failed to add dc=\\username to group dc=foo. C') - self.assertDictEqual(win_groupadd.adduser('dc=foo', 'dc=\\username'), - {'changes': {'Users Added': []}, 'name': 'dc=foo', - 'comment': comt, 'result': False}) + def test_add_user_already_exists(self): + ''' + Test adding a user that already exists + ''' + members = MagicMock(return_value=['HOST\\steve']) + with patch.object(win_groupadd, '_get_group_object', Mock()), \ + patch.object(win_groupadd, '_get_group_members', members), \ + patch('salt.utils.win_functions.get_sam_name', return_value='HOST\\spongebob'): + self.assertDictEqual( + win_groupadd.adduser('foo', 'spongebob'), + {'changes': {'Users Added': ['spongebob']}, + 'comment': '', + 'name': 'foo', + 'result': True}) + + def test_add_user_error(self): + ''' + Test adding a user and encountering an error + ''' + # Create mock group object + class GroupObj(object): + def Add(self, name): + raise pywintypes.com_error(-2147352567, 'Exception occurred.', (0, None, 'C', None, 0, -2146788248), None) + + groupobj_mock = MagicMock(return_value=GroupObj()) + members = MagicMock(return_value=['HOST\\steve']) + with patch.object(win_groupadd, '_get_group_object', groupobj_mock): + with patch.object(win_groupadd, '_get_group_members', members): + comt = ('Failed to add username to group foo. C') + self.assertDictEqual( + win_groupadd.adduser('foo', 'username'), + {'changes': {'Users Added': []}, + 'name': 'foo', + 'comment': comt, + 'result': False}) # 'deluser' function tests: 1 def test_deluser(self): ''' - Test if it remove a user to a group + Test removing a user from a group ''' - ret = {'changes': {'Users Removed': []}, - 'comment': 'User dc=\\username is not a member of dc=foo', - 'name': 'dc=foo', 'result': None} + # Test removing a user + members = MagicMock(return_value=['HOST\\spongebob']) + with patch.object(win_groupadd, '_get_group_object', Mock()), \ + patch.object(win_groupadd, '_get_group_members', members), \ + patch('salt.utils.win_functions.get_sam_name', return_value='HOST\\spongebob'): + ret = {'changes': {'Users Removed': ['spongebob']}, + 'comment': '', + 'name': 'foo', 'result': True} + self.assertDictEqual(win_groupadd.deluser('foo', 'spongebob'), ret) - self.assertDictEqual(win_groupadd.deluser('dc=foo', 'dc=\\username'), - ret) + def test_deluser_no_user(self): + ''' + Test removing a user from a group and that user is not a member of the + group + ''' + + # Test removing a user that's not in the group + members = MagicMock(return_value=['HOST\\steve']) + with patch.object(win_groupadd, '_get_group_object', Mock()), \ + patch.object(win_groupadd, '_get_group_members', members), \ + patch('salt.utils.win_functions.get_sam_name', return_value='HOST\\spongebob'): + ret = {'changes': {'Users Removed': []}, + 'comment': 'User username is not a member of foo', + 'name': 'foo', 'result': None} + self.assertDictEqual(win_groupadd.deluser('foo', 'username'), ret) + + def test_deluser_error(self): + ''' + Test removing a user and encountering an error + ''' + class GroupObj(object): + def Remove(self, name): + raise pywintypes.com_error(-2147352567, 'Exception occurred.', (0, None, 'C', None, 0, -2146788248), None) + + groupobj_mock = MagicMock(return_value=GroupObj()) + + members = MagicMock(return_value=['HOST\\spongebob']) + with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ + patch.object(win_groupadd, '_get_group_members', members), \ + patch('salt.utils.win_functions.get_sam_name', return_value='HOST\\spongebob'): + comt = ('Failed to remove spongebob from group foo. C') + self.assertDictEqual( + win_groupadd.deluser('foo', 'spongebob'), + {'changes': {'Users Removed': []}, + 'name': 'foo', + 'comment': comt, + 'result': False}) # 'members' function tests: 1 From 5ce14df82c20a3c2e9894f0e7f23c6dbf8463bed Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 23 Oct 2017 13:11:46 -0500 Subject: [PATCH 167/184] Change how members are retrieved in win_groupadd This is a solution that works for the other functions that interact with group objects, making it a better overall solution to be used widely in win_groupadd. This also updates the mocking in the tests such that we use a more comprehensive fake class, and more intelligent mocking of get_sam_name using a lambda. --- salt/modules/win_groupadd.py | 61 ++++------- tests/unit/modules/test_win_groupadd.py | 136 +++++++++++++----------- 2 files changed, 98 insertions(+), 99 deletions(-) diff --git a/salt/modules/win_groupadd.py b/salt/modules/win_groupadd.py index 322f172fd2..02c7cd74cc 100644 --- a/salt/modules/win_groupadd.py +++ b/salt/modules/win_groupadd.py @@ -42,6 +42,20 @@ def _get_computer_object(): return nt.GetObject('', 'WinNT://.,computer') +def _get_group_object(name): + pythoncom.CoInitialize() + nt = win32com.client.Dispatch('AdsNameSpaces') + return nt.GetObject('', 'WinNT://./' + name + ',group') + + +def _get_username(member): + ''' + Resolve the username from the member object returned from a group query + ''' + return member.ADSPath.replace('WinNT://', '').replace( + '/', '\\').encode('ascii', 'backslashreplace').lower() + + def add(name, **kwargs): ''' Add the specified group @@ -147,14 +161,9 @@ def info(name): salt '*' group.info foo ''' groupObj = _get_group_object(name) - existing_members = _get_group_members(groupObj) try: gr_name = groupObj.Name - gr_mem = [] - for member in existing_members: - gr_mem.append( - member.ADSPath.replace('WinNT://', '').replace( - '/', '\\').encode('ascii', 'backslashreplace')) + gr_mem = [_get_username(x) for x in groupObj.members()] except pywintypes.com_error: return False @@ -199,13 +208,8 @@ def getent(refresh=False): results = nt.GetObject('', 'WinNT://.') results.Filter = ['group'] for result in results: - member_list = [] - for member in result.members(): - member_list.append( - member.AdsPath.replace('WinNT://', '').replace( - '/', '\\').encode('ascii', 'backslashreplace')) group = {'gid': __salt__['file.group_to_gid'](result.name), - 'members': member_list, + 'members': [_get_username(x) for x in result.members()], 'name': result.name, 'passwd': 'x'} ret.append(group) @@ -213,21 +217,6 @@ def getent(refresh=False): return ret -def _get_group_object(name): - pythoncom.CoInitialize() - nt = win32com.client.Dispatch('AdsNameSpaces') - return nt.GetObject('', 'WinNT://./' + name + ',group') - - -def _get_group_members(groupObj): - existingMembers = [] - for member in groupObj.members(): - existingMembers.append( - member.ADSPath.replace('WinNT://', '').replace( - '/', '\\').encode('ascii', 'backslashreplace').lower()) - return existingMembers - - def adduser(name, username, **kwargs): ''' Add a user to a group @@ -256,10 +245,11 @@ def adduser(name, username, **kwargs): 'comment': ''} groupObj = _get_group_object(name) - existingMembers = _get_group_members(groupObj) + existingMembers = [_get_username(x) for x in groupObj.members()] + username = salt.utils.win_functions.get_sam_name(username) try: - if salt.utils.win_functions.get_sam_name(username) not in existingMembers: + if username not in existingMembers: if not __opts__['test']: groupObj.Add('WinNT://' + username.replace('\\', '/')) @@ -309,7 +299,7 @@ def deluser(name, username, **kwargs): 'comment': ''} groupObj = _get_group_object(name) - existingMembers = _get_group_members(groupObj) + existingMembers = [_get_username(x) for x in groupObj.members()] try: if salt.utils.win_functions.get_sam_name(username) in existingMembers: @@ -368,10 +358,8 @@ def members(name, members_list, **kwargs): ret['comment'].append('Members is not a list object') return ret - pythoncom.CoInitialize() - nt = win32com.client.Dispatch('AdsNameSpaces') try: - groupObj = nt.GetObject('', 'WinNT://./' + name + ',group') + groupObj = _get_group_object(name) except pywintypes.com_error as com_err: if len(com_err.excepinfo) >= 2: friendly_error = com_err.excepinfo[2].rstrip('\r\n') @@ -380,12 +368,7 @@ def members(name, members_list, **kwargs): 'Failure accessing group {0}. {1}' ).format(name, friendly_error)) return ret - existingMembers = [] - for member in groupObj.members(): - existingMembers.append( - member.ADSPath.replace('WinNT://', '').replace( - '/', '\\').encode('ascii', 'backslashreplace').lower()) - + existingMembers = [_get_username(x) for x in groupObj.members()] existingMembers.sort() members_list.sort() diff --git a/tests/unit/modules/test_win_groupadd.py b/tests/unit/modules/test_win_groupadd.py index 89426b21a0..9f9af18f53 100644 --- a/tests/unit/modules/test_win_groupadd.py +++ b/tests/unit/modules/test_win_groupadd.py @@ -19,6 +19,7 @@ from tests.support.mock import ( # Import Salt Libs import salt.modules.win_groupadd as win_groupadd +import salt.utils.win_functions # Import Other Libs # pylint: disable=unused-import @@ -32,6 +33,37 @@ except ImportError: # pylint: enable=unused-import +class MockMember(object): + def __init__(self, name): + self.ADSPath = name + + +class MockGroupObj(object): + def __init__(self, ads_users): + self._members = [MockMember(x) for x in ads_users] + + def members(self): + return self._members + + def Add(self): + ''' + This should be a no-op unless we want to test raising an error, in + which case this should be overridden in a subclass. + ''' + pass + + def Remove(self): + ''' + This should be a no-op unless we want to test raising an error, in + which case this should be overridden in a subclass. + ''' + pass + + +if not NO_MOCK: + sam_mock = MagicMock(side_effect=lambda x: 'HOST\\' + x) + + @skipIf(not HAS_WIN_LIBS, 'win_groupadd unit tests can only be run if win32com, pythoncom, and pywintypes are installed') @skipIf(NO_MOCK, NO_MOCK_REASON) class WinGroupTestCase(TestCase, LoaderModuleMockMixin): @@ -151,21 +183,14 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): ''' Test if it return information about a group. ''' - members = MagicMock(return_value=['HOST\\steve']) - with patch.object(win_groupadd, '_get_group_object', Mock()), \ - patch.object(win_groupadd, '_get_group_members', members): + groupobj_mock = MagicMock(return_value=MockGroupObj(['WinNT://HOST/steve'])) + with patch.object(win_groupadd, '_get_group_object', groupobj_mock): self.assertDictEqual(win_groupadd.info('salt'), {'gid': None, - 'members': ['user1'], + 'members': ['HOST\\steve'], 'passwd': None, 'name': 'salt'}) - with patch(win_groupadd.win32.client, 'flag', 1): - self.assertFalse(win_groupadd.info('dc=salt')) - - with patch(win_groupadd.win32.client, 'flag', 2): - self.assertFalse(win_groupadd.info('dc=salt')) - # 'getent' function tests: 1 def test_getent(self): @@ -175,73 +200,70 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(win_groupadd.__context__, {'group.getent': True}): self.assertTrue(win_groupadd.getent()) - # 'adduser' function tests: 1 + # 'adduser' function tests: 3 def test_adduser(self): ''' Test adding a user to a group ''' - members = MagicMock(return_value=['HOST\\steve']) - with patch.object(win_groupadd, '_get_group_object', Mock()),\ - patch.object(win_groupadd, '_get_group_members', members),\ - patch('salt.utils.win_functions.get_sam_name', return_value='HOST\\spongebob'): + groupobj_mock = MagicMock(return_value=MockGroupObj(['WinNT://HOST/steve'])) + with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ + patch.object(salt.utils.win_functions, 'get_sam_name', sam_mock): self.assertDictEqual( win_groupadd.adduser('foo', 'spongebob'), - {'changes': {'Users Added': ['spongebob']}, + {'changes': {'Users Added': ['HOST\\spongebob']}, 'comment': '', 'name': 'foo', 'result': True}) - def test_add_user_already_exists(self): + def test_adduser_already_exists(self): ''' Test adding a user that already exists ''' - members = MagicMock(return_value=['HOST\\steve']) - with patch.object(win_groupadd, '_get_group_object', Mock()), \ - patch.object(win_groupadd, '_get_group_members', members), \ - patch('salt.utils.win_functions.get_sam_name', return_value='HOST\\spongebob'): + groupobj_mock = MagicMock(return_value=MockGroupObj(['WinNT://HOST/steve'])) + with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ + patch.object(salt.utils.win_functions, 'get_sam_name', sam_mock): self.assertDictEqual( - win_groupadd.adduser('foo', 'spongebob'), - {'changes': {'Users Added': ['spongebob']}, - 'comment': '', + win_groupadd.adduser('foo', 'HOST\\steve'), + {'changes': {'Users Added': []}, + 'comment': 'User HOST\\steve is already a member of foo', 'name': 'foo', - 'result': True}) + 'result': None}) - def test_add_user_error(self): + def test_adduser_error(self): ''' Test adding a user and encountering an error ''' - # Create mock group object - class GroupObj(object): + # Create mock group object with mocked Add function which raises the + # exception we need in order to test the error case. + class GroupObj(MockGroupObj): def Add(self, name): raise pywintypes.com_error(-2147352567, 'Exception occurred.', (0, None, 'C', None, 0, -2146788248), None) - groupobj_mock = MagicMock(return_value=GroupObj()) - members = MagicMock(return_value=['HOST\\steve']) - with patch.object(win_groupadd, '_get_group_object', groupobj_mock): - with patch.object(win_groupadd, '_get_group_members', members): - comt = ('Failed to add username to group foo. C') - self.assertDictEqual( - win_groupadd.adduser('foo', 'username'), - {'changes': {'Users Added': []}, - 'name': 'foo', - 'comment': comt, - 'result': False}) + groupobj_mock = MagicMock(return_value=GroupObj(['WinNT://HOST/steve'])) + with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ + patch.object(salt.utils.win_functions, 'get_sam_name', sam_mock): + self.assertDictEqual( + win_groupadd.adduser('foo', 'username'), + {'changes': {'Users Added': []}, + 'name': 'foo', + 'comment': 'Failed to add HOST\\username to group foo. C', + 'result': False}) - # 'deluser' function tests: 1 + # 'deluser' function tests: 3 def test_deluser(self): ''' Test removing a user from a group ''' # Test removing a user - members = MagicMock(return_value=['HOST\\spongebob']) - with patch.object(win_groupadd, '_get_group_object', Mock()), \ - patch.object(win_groupadd, '_get_group_members', members), \ - patch('salt.utils.win_functions.get_sam_name', return_value='HOST\\spongebob'): + groupobj_mock = MagicMock(return_value=MockGroupObj(['WinNT://HOST/spongebob'])) + with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ + patch.object(salt.utils.win_functions, 'get_sam_name', sam_mock): ret = {'changes': {'Users Removed': ['spongebob']}, 'comment': '', - 'name': 'foo', 'result': True} + 'name': 'foo', + 'result': True} self.assertDictEqual(win_groupadd.deluser('foo', 'spongebob'), ret) def test_deluser_no_user(self): @@ -249,15 +271,13 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): Test removing a user from a group and that user is not a member of the group ''' - - # Test removing a user that's not in the group - members = MagicMock(return_value=['HOST\\steve']) - with patch.object(win_groupadd, '_get_group_object', Mock()), \ - patch.object(win_groupadd, '_get_group_members', members), \ - patch('salt.utils.win_functions.get_sam_name', return_value='HOST\\spongebob'): + groupobj_mock = MagicMock(return_value=MockGroupObj(['WinNT://HOST/steve'])) + with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ + patch.object(salt.utils.win_functions, 'get_sam_name', sam_mock): ret = {'changes': {'Users Removed': []}, - 'comment': 'User username is not a member of foo', - 'name': 'foo', 'result': None} + 'comment': 'User HOST\\spongebob is not a member of foo', + 'name': 'foo', + 'result': None} self.assertDictEqual(win_groupadd.deluser('foo', 'username'), ret) def test_deluser_error(self): @@ -268,18 +288,14 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): def Remove(self, name): raise pywintypes.com_error(-2147352567, 'Exception occurred.', (0, None, 'C', None, 0, -2146788248), None) - groupobj_mock = MagicMock(return_value=GroupObj()) - - members = MagicMock(return_value=['HOST\\spongebob']) + groupobj_mock = MagicMock(return_value=GroupObj(['WinNT://HOST/spongebob'])) with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ - patch.object(win_groupadd, '_get_group_members', members), \ - patch('salt.utils.win_functions.get_sam_name', return_value='HOST\\spongebob'): - comt = ('Failed to remove spongebob from group foo. C') + patch.object(salt.utils.win_functions, 'get_sam_name', sam_mock): self.assertDictEqual( win_groupadd.deluser('foo', 'spongebob'), {'changes': {'Users Removed': []}, 'name': 'foo', - 'comment': comt, + 'comment': 'Failed to remove spongebob from group foo. C', 'result': False}) # 'members' function tests: 1 From 7a3ff9387dcec1e8dca410facf64c6ec4fb9ca4a Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 23 Oct 2017 15:17:42 -0600 Subject: [PATCH 168/184] Mock the rest of the tests --- salt/modules/win_groupadd.py | 56 ++++--- tests/unit/modules/test_win_groupadd.py | 196 +++++++++++++++++++----- 2 files changed, 194 insertions(+), 58 deletions(-) diff --git a/salt/modules/win_groupadd.py b/salt/modules/win_groupadd.py index 02c7cd74cc..d25bc1fa60 100644 --- a/salt/modules/win_groupadd.py +++ b/salt/modules/win_groupadd.py @@ -53,7 +53,7 @@ def _get_username(member): Resolve the username from the member object returned from a group query ''' return member.ADSPath.replace('WinNT://', '').replace( - '/', '\\').encode('ascii', 'backslashreplace').lower() + '/', '\\').encode('ascii', 'backslashreplace') def add(name, **kwargs): @@ -160,8 +160,8 @@ def info(name): salt '*' group.info foo ''' - groupObj = _get_group_object(name) try: + groupObj = _get_group_object(name) gr_name = groupObj.Name gr_mem = [_get_username(x) for x in groupObj.members()] except pywintypes.com_error: @@ -202,15 +202,12 @@ def getent(refresh=False): ret = [] - pythoncom.CoInitialize() - nt = win32com.client.Dispatch('AdsNameSpaces') + results = _get_all_groups() - results = nt.GetObject('', 'WinNT://.') - results.Filter = ['group'] for result in results: - group = {'gid': __salt__['file.group_to_gid'](result.name), + group = {'gid': __salt__['file.group_to_gid'](result.Name), 'members': [_get_username(x) for x in result.members()], - 'name': result.name, + 'name': result.Name, 'passwd': 'x'} ret.append(group) __context__['group.getent'] = ret @@ -244,7 +241,16 @@ def adduser(name, username, **kwargs): 'changes': {'Users Added': []}, 'comment': ''} - groupObj = _get_group_object(name) + try: + groupObj = _get_group_object(name) + except pywintypes.com_error as com_err: + if len(com_err.excepinfo) >= 2: + friendly_error = com_err.excepinfo[2].rstrip('\r\n') + ret['result'] = False + ret['comment'] = 'Failure accessing group {0}. {1}' \ + ''.format(name, friendly_error) + return ret + existingMembers = [_get_username(x) for x in groupObj.members()] username = salt.utils.win_functions.get_sam_name(username) @@ -298,7 +304,16 @@ def deluser(name, username, **kwargs): 'changes': {'Users Removed': []}, 'comment': ''} - groupObj = _get_group_object(name) + try: + groupObj = _get_group_object(name) + except pywintypes.com_error as com_err: + if len(com_err.excepinfo) >= 2: + friendly_error = com_err.excepinfo[2].rstrip('\r\n') + ret['result'] = False + ret['comment'] = 'Failure accessing group {0}. {1}' \ + ''.format(name, friendly_error) + return ret + existingMembers = [_get_username(x) for x in groupObj.members()] try: @@ -412,6 +427,15 @@ def members(name, members_list, **kwargs): return ret +def _get_all_groups(): + pythoncom.CoInitialize() + nt = win32com.client.Dispatch('AdsNameSpaces') + + results = nt.GetObject('', 'WinNT://.') + results.Filter = ['group'] + return results + + def list_groups(refresh=False): ''' Return a list of groups @@ -434,18 +458,14 @@ def list_groups(refresh=False): salt '*' group.list_groups ''' if 'group.list_groups' in __context__ and not refresh: - return __context__['group.getent'] + return __context__['group.list_groups'] + + results = _get_all_groups() ret = [] - pythoncom.CoInitialize() - nt = win32com.client.Dispatch('AdsNameSpaces') - - results = nt.GetObject('', 'WinNT://.') - results.Filter = ['group'] - for result in results: - ret.append(result.name) + ret.append(result.Name) __context__['group.list_groups'] = ret diff --git a/tests/unit/modules/test_win_groupadd.py b/tests/unit/modules/test_win_groupadd.py index 9f9af18f53..a260898ca6 100644 --- a/tests/unit/modules/test_win_groupadd.py +++ b/tests/unit/modules/test_win_groupadd.py @@ -27,6 +27,8 @@ try: import win32com import pythoncom import pywintypes + PYWINTYPES_ERROR = pywintypes.com_error( + -1234, 'Exception occurred.', (0, None, 'C', None, 0, -4321), None) HAS_WIN_LIBS = True except ImportError: HAS_WIN_LIBS = False @@ -39,20 +41,21 @@ class MockMember(object): class MockGroupObj(object): - def __init__(self, ads_users): + def __init__(self, ads_name, ads_users): self._members = [MockMember(x) for x in ads_users] + self.Name = ads_name def members(self): return self._members - def Add(self): + def Add(self, name): ''' This should be a no-op unless we want to test raising an error, in which case this should be overridden in a subclass. ''' pass - def Remove(self): + def Remove(self, name): ''' This should be a no-op unless we want to test raising an error, in which case this should be overridden in a subclass. @@ -110,7 +113,7 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): ''' class CompObj(object): def Create(self, type, name): - raise pywintypes.com_error(-2147352567, 'Exception occurred.', (0, None, 'C', None, 0, -2146788248), None) + raise PYWINTYPES_ERROR compobj_mock = MagicMock(return_value=CompObj()) @@ -159,7 +162,7 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): ''' class CompObj(object): def Delete(self, type, name): - raise pywintypes.com_error(-2147352567, 'Exception occurred.', (0, None, 'C', None, 0, -2146788248), None) + raise PYWINTYPES_ERROR compobj_mock = MagicMock(return_value=CompObj()) @@ -183,7 +186,7 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): ''' Test if it return information about a group. ''' - groupobj_mock = MagicMock(return_value=MockGroupObj(['WinNT://HOST/steve'])) + groupobj_mock = MagicMock(return_value=MockGroupObj('salt', ['WinNT://HOST/steve'])) with patch.object(win_groupadd, '_get_group_object', groupobj_mock): self.assertDictEqual(win_groupadd.info('salt'), {'gid': None, @@ -191,22 +194,33 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): 'passwd': None, 'name': 'salt'}) - # 'getent' function tests: 1 - def test_getent(self): + groupobj_mock = MagicMock( + return_value=[ + MockGroupObj('salt', ['WinNT://HOST/steve']), + MockGroupObj('salty', ['WinNT://HOST/spongebob'])]) + mock_g_to_g = MagicMock(side_effect=[1,2,3,4,5]) + with patch.object(win_groupadd, '_get_all_groups', groupobj_mock),\ + patch.dict(win_groupadd.__salt__, {'file.group_to_gid': mock_g_to_g}): + self.assertListEqual( + win_groupadd.getent(), + [ + {'gid': 1, 'members': ['HOST\\steve'], 'name': 'salt', 'passwd': 'x'}, + {'gid': 2, 'members': ['HOST\\spongebob'], 'name': 'salty', 'passwd': 'x'} + ]) + + def test_getent_context(self): ''' - Test if it return info on all groups + Test group.getent is using the values in __context__ ''' with patch.dict(win_groupadd.__context__, {'group.getent': True}): self.assertTrue(win_groupadd.getent()) - # 'adduser' function tests: 3 - def test_adduser(self): ''' Test adding a user to a group ''' - groupobj_mock = MagicMock(return_value=MockGroupObj(['WinNT://HOST/steve'])) + groupobj_mock = MagicMock(return_value=MockGroupObj('foo', ['WinNT://HOST/steve'])) with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ patch.object(salt.utils.win_functions, 'get_sam_name', sam_mock): self.assertDictEqual( @@ -220,11 +234,11 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): ''' Test adding a user that already exists ''' - groupobj_mock = MagicMock(return_value=MockGroupObj(['WinNT://HOST/steve'])) + groupobj_mock = MagicMock(return_value=MockGroupObj('foo', ['WinNT://HOST/steve'])) with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ patch.object(salt.utils.win_functions, 'get_sam_name', sam_mock): self.assertDictEqual( - win_groupadd.adduser('foo', 'HOST\\steve'), + win_groupadd.adduser('foo', 'steve'), {'changes': {'Users Added': []}, 'comment': 'User HOST\\steve is already a member of foo', 'name': 'foo', @@ -238,9 +252,9 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): # exception we need in order to test the error case. class GroupObj(MockGroupObj): def Add(self, name): - raise pywintypes.com_error(-2147352567, 'Exception occurred.', (0, None, 'C', None, 0, -2146788248), None) + raise PYWINTYPES_ERROR - groupobj_mock = MagicMock(return_value=GroupObj(['WinNT://HOST/steve'])) + groupobj_mock = MagicMock(return_value=GroupObj('foo', ['WinNT://HOST/steve'])) with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ patch.object(salt.utils.win_functions, 'get_sam_name', sam_mock): self.assertDictEqual( @@ -250,6 +264,18 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): 'comment': 'Failed to add HOST\\username to group foo. C', 'result': False}) + + def test_adduser_group_does_not_exist(self): + groupobj_mock = MagicMock(side_effect=PYWINTYPES_ERROR) + with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ + patch.object(salt.utils.win_functions, 'get_sam_name', sam_mock): + self.assertDictEqual( + win_groupadd.adduser('foo', 'spongebob'), + {'changes': {'Users Added': []}, + 'name': 'foo', + 'comment': 'Failure accessing group foo. C', + 'result': False}) + # 'deluser' function tests: 3 def test_deluser(self): @@ -257,7 +283,7 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): Test removing a user from a group ''' # Test removing a user - groupobj_mock = MagicMock(return_value=MockGroupObj(['WinNT://HOST/spongebob'])) + groupobj_mock = MagicMock(return_value=MockGroupObj('foo', ['WinNT://HOST/spongebob'])) with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ patch.object(salt.utils.win_functions, 'get_sam_name', sam_mock): ret = {'changes': {'Users Removed': ['spongebob']}, @@ -271,24 +297,24 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): Test removing a user from a group and that user is not a member of the group ''' - groupobj_mock = MagicMock(return_value=MockGroupObj(['WinNT://HOST/steve'])) + groupobj_mock = MagicMock(return_value=MockGroupObj('foo', ['WinNT://HOST/steve'])) with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ patch.object(salt.utils.win_functions, 'get_sam_name', sam_mock): ret = {'changes': {'Users Removed': []}, - 'comment': 'User HOST\\spongebob is not a member of foo', + 'comment': 'User spongebob is not a member of foo', 'name': 'foo', 'result': None} - self.assertDictEqual(win_groupadd.deluser('foo', 'username'), ret) + self.assertDictEqual(win_groupadd.deluser('foo', 'spongebob'), ret) def test_deluser_error(self): ''' Test removing a user and encountering an error ''' - class GroupObj(object): + class GroupObj(MockGroupObj): def Remove(self, name): - raise pywintypes.com_error(-2147352567, 'Exception occurred.', (0, None, 'C', None, 0, -2146788248), None) + raise PYWINTYPES_ERROR - groupobj_mock = MagicMock(return_value=GroupObj(['WinNT://HOST/spongebob'])) + groupobj_mock = MagicMock(return_value=GroupObj('foo', ['WinNT://HOST/spongebob'])) with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ patch.object(salt.utils.win_functions, 'get_sam_name', sam_mock): self.assertDictEqual( @@ -298,28 +324,118 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): 'comment': 'Failed to remove spongebob from group foo. C', 'result': False}) + def test_deluser_group_does_not_exist(self): + groupobj_mock = MagicMock(side_effect=PYWINTYPES_ERROR) + with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ + patch.object(salt.utils.win_functions, 'get_sam_name', sam_mock): + self.assertDictEqual( + win_groupadd.deluser('foo', 'spongebob'), + {'changes': {'Users Removed': []}, + 'name': 'foo', + 'comment': 'Failure accessing group foo. C', + 'result': False}) + # 'members' function tests: 1 def test_members(self): ''' - Test if it remove a user to a group + Test adding a list of members to a group, all existing users removed ''' - comment = ['Failure accessing group dc=foo. C'] - ret = {'name': 'dc=foo', 'result': False, 'comment': comment, - 'changes': {'Users Added': [], 'Users Removed': []}} + groupobj_mock = MagicMock(return_value=MockGroupObj('foo', ['WinNT://HOST/steve'])) + with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ + patch.object(salt.utils.win_functions, 'get_sam_name', sam_mock): + self.assertDictEqual( + win_groupadd.members('foo', 'spongebob,patrick,squidward'), + {'changes': { + 'Users Added': ['HOST\\patrick', 'HOST\\spongebob', 'HOST\\squidward'], + 'Users Removed': ['HOST\\steve'] + }, + 'comment': [], + 'name': 'foo', + 'result': True}) - with patch(win_groupadd.win32.client, 'flag', 2): - self.assertDictEqual(win_groupadd.members - ('dc=foo', 'dc=\\user1,dc=\\user2,dc=\\user3'), - ret) + def test_members_correct_membership(self): + ''' + Test adding a list of users where the list of users already exists + ''' + members_list = ['WinNT://HOST/spongebob', + 'WinNT://HOST/squidward', + 'WinNT://HOST/patrick'] + groupobj_mock = MagicMock(return_value=MockGroupObj('foo', members_list)) + with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ + patch.object(salt.utils.win_functions, 'get_sam_name', sam_mock): + self.assertDictEqual( + win_groupadd.members('foo', 'spongebob,patrick,squidward'), + {'changes': {'Users Added': [], 'Users Removed': []}, + 'comment': ['foo membership is correct'], + 'name': 'foo', + 'result': None}) - with patch(win_groupadd.win32.client, 'flag', 1): - comment = ['Failed to add dc=\\user2 to dc=foo. C', - 'Failed to remove dc=\\user1 from dc=foo. C'] - ret.update({'comment': comment, 'result': False}) - self.assertDictEqual(win_groupadd.members('dc=foo', 'dc=\\user2'), ret) + def test_members_group_does_not_exist(self): + ''' + Test adding a list of users where the group does not exist + ''' + groupobj_mock = MagicMock(side_effect=PYWINTYPES_ERROR) + with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ + patch.object(salt.utils.win_functions, 'get_sam_name', sam_mock): + self.assertDictEqual( + win_groupadd.members('foo', 'spongebob'), + {'changes': {'Users Added': [], 'Users Removed': []}, + 'comment': ['Failure accessing group foo. C'], + 'name': 'foo', + 'result': False}) - with patch(win_groupadd.win32.client, 'flag', None): - comment = ['dc=foo membership is correct'] - ret.update({'comment': comment, 'result': None}) - self.assertDictEqual(win_groupadd.members('dc=foo', 'dc=\\user1'), ret) + def test_members_fail_to_remove(self): + ''' + Test adding a list of members and fail to remove members not in the list + ''' + class GroupObj(MockGroupObj): + def Remove(self, name): + raise PYWINTYPES_ERROR + + groupobj_mock = MagicMock(return_value=GroupObj('foo', ['WinNT://HOST/spongebob'])) + with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ + patch.object(salt.utils.win_functions, 'get_sam_name', sam_mock): + self.assertDictEqual( + win_groupadd.members('foo', 'patrick'), + {'changes': {'Users Added': ['HOST\\patrick'], 'Users Removed': []}, + 'comment': ['Failed to remove HOST\\spongebob from foo. C'], + 'name': 'foo', + 'result': False}) + + def test_members_fail_to_add(self): + ''' + Test adding a list of members and failing to add + ''' + class GroupObj(MockGroupObj): + def Add(self, name): + raise PYWINTYPES_ERROR + + groupobj_mock = MagicMock(return_value=GroupObj('foo', ['WinNT://HOST/spongebob'])) + with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ + patch.object(salt.utils.win_functions, 'get_sam_name', sam_mock): + self.assertDictEqual( + win_groupadd.members('foo', 'patrick'), + {'changes': {'Users Added': [], 'Users Removed': ['HOST\\spongebob']}, + 'comment': ['Failed to add HOST\\patrick to foo. C'], + 'name': 'foo', + 'result': False}) + + def test_list_groups(self): + ''' + Test that list groups returns a list of groups by name + ''' + groupobj_mock = MagicMock( + return_value=[ + MockGroupObj('salt', ['WinNT://HOST/steve']), + MockGroupObj('salty', ['WinNT://HOST/Administrator'])]) + with patch.object(win_groupadd, '_get_all_groups', groupobj_mock): + self.assertListEqual(win_groupadd.list_groups(), + ['salt', 'salty']) + + def test_list_groups_context(self): + ''' + Test group.list_groups is using the values in __context__ + ''' + with patch.dict(win_groupadd.__context__, {'group.list_groups': True}): + self.assertTrue(win_groupadd.list_groups()) From b0caec320e0cf40092411f43317eca39bc090f16 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 23 Oct 2017 15:25:13 -0600 Subject: [PATCH 169/184] Move _get_all_groups up to the top --- salt/modules/win_groupadd.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/salt/modules/win_groupadd.py b/salt/modules/win_groupadd.py index d25bc1fa60..441da79a5c 100644 --- a/salt/modules/win_groupadd.py +++ b/salt/modules/win_groupadd.py @@ -48,6 +48,15 @@ def _get_group_object(name): return nt.GetObject('', 'WinNT://./' + name + ',group') +def _get_all_groups(): + pythoncom.CoInitialize() + nt = win32com.client.Dispatch('AdsNameSpaces') + + results = nt.GetObject('', 'WinNT://.') + results.Filter = ['group'] + return results + + def _get_username(member): ''' Resolve the username from the member object returned from a group query @@ -427,15 +436,6 @@ def members(name, members_list, **kwargs): return ret -def _get_all_groups(): - pythoncom.CoInitialize() - nt = win32com.client.Dispatch('AdsNameSpaces') - - results = nt.GetObject('', 'WinNT://.') - results.Filter = ['group'] - return results - - def list_groups(refresh=False): ''' Return a list of groups From 1f44d8d5e6ba78f387dea8ef86687497f74249df Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 23 Oct 2017 15:30:43 -0600 Subject: [PATCH 170/184] Document helper functions --- salt/modules/win_groupadd.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/salt/modules/win_groupadd.py b/salt/modules/win_groupadd.py index 441da79a5c..9c0307265b 100644 --- a/salt/modules/win_groupadd.py +++ b/salt/modules/win_groupadd.py @@ -37,21 +37,43 @@ def __virtual__(): def _get_computer_object(): + ''' + A helper function to get the object for the local machine + + Returns: + object: Returns the computer object for the local machine + ''' pythoncom.CoInitialize() nt = win32com.client.Dispatch('AdsNameSpaces') return nt.GetObject('', 'WinNT://.,computer') def _get_group_object(name): + ''' + A helper function to get a specified group object + + Args: + + name (str): The name of the object + + Returns: + object: The specified group object + ''' pythoncom.CoInitialize() nt = win32com.client.Dispatch('AdsNameSpaces') return nt.GetObject('', 'WinNT://./' + name + ',group') def _get_all_groups(): + ''' + A helper function that gets a list of group objects for all groups on the + machine + + Returns: + iter: A list of objects for all groups on the machine + ''' pythoncom.CoInitialize() nt = win32com.client.Dispatch('AdsNameSpaces') - results = nt.GetObject('', 'WinNT://.') results.Filter = ['group'] return results @@ -60,6 +82,9 @@ def _get_all_groups(): def _get_username(member): ''' Resolve the username from the member object returned from a group query + + Returns: + str: The username converted to domain\\username format ''' return member.ADSPath.replace('WinNT://', '').replace( '/', '\\').encode('ascii', 'backslashreplace') From 609361bf485e172855e51982fb324685dd71df02 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 24 Oct 2017 09:27:21 -0600 Subject: [PATCH 171/184] Fix some lint errors --- tests/unit/modules/test_win_groupadd.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/tests/unit/modules/test_win_groupadd.py b/tests/unit/modules/test_win_groupadd.py index a260898ca6..7404f76e24 100644 --- a/tests/unit/modules/test_win_groupadd.py +++ b/tests/unit/modules/test_win_groupadd.py @@ -78,8 +78,6 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): win_groupadd: {'__opts__': {'test': False}} } - # 'add' function tests: 1 - def test_add(self): ''' Test adding a new group @@ -126,8 +124,6 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): 'result': False, 'comment': 'Failed to create group foo. C'}) - # 'delete' function tests: 1 - def test_delete(self): ''' Test removing a group @@ -179,9 +175,6 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): 'result': False, 'comment': 'Failed to remove group foo. C'}) - - # 'info' function tests: 1 - def test_info(self): ''' Test if it return information about a group. @@ -199,7 +192,7 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): return_value=[ MockGroupObj('salt', ['WinNT://HOST/steve']), MockGroupObj('salty', ['WinNT://HOST/spongebob'])]) - mock_g_to_g = MagicMock(side_effect=[1,2,3,4,5]) + mock_g_to_g = MagicMock(side_effect=[1, 2]) with patch.object(win_groupadd, '_get_all_groups', groupobj_mock),\ patch.dict(win_groupadd.__salt__, {'file.group_to_gid': mock_g_to_g}): self.assertListEqual( @@ -264,7 +257,6 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): 'comment': 'Failed to add HOST\\username to group foo. C', 'result': False}) - def test_adduser_group_does_not_exist(self): groupobj_mock = MagicMock(side_effect=PYWINTYPES_ERROR) with patch.object(win_groupadd, '_get_group_object', groupobj_mock), \ @@ -276,8 +268,6 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): 'comment': 'Failure accessing group foo. C', 'result': False}) - # 'deluser' function tests: 3 - def test_deluser(self): ''' Test removing a user from a group @@ -335,8 +325,6 @@ class WinGroupTestCase(TestCase, LoaderModuleMockMixin): 'comment': 'Failure accessing group foo. C', 'result': False}) - # 'members' function tests: 1 - def test_members(self): ''' Test adding a list of members to a group, all existing users removed From 2ffcc447211a40e57f5b766b8af33befb815fcce Mon Sep 17 00:00:00 2001 From: Ollie Armstrong Date: Tue, 24 Oct 2017 17:32:02 +0100 Subject: [PATCH 172/184] docker_container.running sort list of links When the docker_container.running module is comparing the defined state and the current state of a container, the list of linked containers in the current state is is a non-deterministic order. This results in a container recreation even though the links are the same (just in a different order). This patches the comparison function to lexically sort both the existing and desired list of links, making the comparison deterministic and not recreating a container when the links have not changed. Fixes #44258. --- salt/modules/dockermod.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index 6249988e27..dadb3c8c17 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -915,8 +915,8 @@ def compare_container(first, second, ignore=None): ret.setdefault(conf_dict, {})[item] = {'old': image1, 'new': image2} else: if item == 'Links': - val1 = _scrub_links(val1, first) - val2 = _scrub_links(val2, second) + val1 = sorted(_scrub_links(val1, first)) + val2 = sorted(_scrub_links(val2, second)) if val1 != val2: ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} # Check for optionally-present items that were in the second container @@ -938,8 +938,8 @@ def compare_container(first, second, ignore=None): ret.setdefault(conf_dict, {})[item] = {'old': image1, 'new': image2} else: if item == 'Links': - val1 = _scrub_links(val1, first) - val2 = _scrub_links(val2, second) + val1 = sorted(_scrub_links(val1, first)) + val2 = sorted(_scrub_links(val2, second)) if val1 != val2: ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} return ret From e5701b472d7b999a09ebda97bb11dfe6c0e36480 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Tue, 24 Oct 2017 16:59:56 -0400 Subject: [PATCH 173/184] Add state, grains and service proxy tests --- tests/integration/proxy/test_simple.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/integration/proxy/test_simple.py b/tests/integration/proxy/test_simple.py index dbca73bafa..e1af027cf2 100644 --- a/tests/integration/proxy/test_simple.py +++ b/tests/integration/proxy/test_simple.py @@ -67,3 +67,23 @@ class ProxyMinionSimpleTestCase(ModuleCase): ret = self.run_function('service.start', ['samba'], minion_tgt='proxytest') ret = self.run_function('service.status', ['samba'], minion_tgt='proxytest') self.assertTrue(ret) + + def test_service_get_all(self): + ret = self.run_function('service.get_all', minion_tgt='proxytest') + self.assertTrue(ret) + self.assertIn('samba', ' '.join(ret)) + + def test_grains_items(self): + ret = self.run_function('grains.items', minion_tgt='proxytest') + self.assertEqual(ret['kernel'], 'proxy') + self.assertEqual(ret['kernelrelease'], 'proxy') + + def test_state_apply(self): + ret = self.run_function('state.apply', ['core'], minion_tgt='proxytest') + for key, value in ret.items(): + self.assertTrue(value['result']) + + def test_state_highstate(self): + ret = self.run_function('state.highstate', minion_tgt='proxytest') + for key, value in ret.items(): + self.assertTrue(value['result']) From e7aebf46ceef07e7a79455ad26fc78f1b221dcd5 Mon Sep 17 00:00:00 2001 From: Federico Pires Date: Wed, 25 Oct 2017 10:58:12 -0300 Subject: [PATCH 174/184] Support softlayer dedicated host in salt-cloud. Update docs as well. --- doc/topics/cloud/softlayer.rst | 18 ++++++++++++++++-- salt/cloud/clouds/softlayer.py | 6 ++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/doc/topics/cloud/softlayer.rst b/doc/topics/cloud/softlayer.rst index 468dca283d..d073f39b98 100644 --- a/doc/topics/cloud/softlayer.rst +++ b/doc/topics/cloud/softlayer.rst @@ -94,6 +94,8 @@ Set up an initial profile at ``/etc/salt/cloud.profiles``: private_vlan: 396 private_network: True private_ssh: True + # Use a dedicated host instead of cloud + dedicated_host_id: 1234 # May be used _instead_of_ image global_identifier: 320d8be5-46c0-dead-cafe-13e3c51 @@ -334,9 +336,21 @@ it can be verified with Salt: # salt 'myserver.example.com' test.ping - -Cloud Profiles +Dedicated Host ~~~~~~~~~~~~~~ +Soflayer allows the creation of new VMs in a dedicated host. This means that +you can order and pay a fixed amount for a bare metal dedicated host and use +it to provision as many VMs as you can fit in there. If you want your VMs to +be launched in a dedicated host, instead of Sofltayer's cloud, set the +``dedicated_host_id`` parameter in your profile. + +dedicated_host_id +----------------- +The id of the dedicated host where the VMs should be created. If not set, VMs +will be created in Softlayer's cloud instead. + +Bare metal Profiles +~~~~~~~~~~~~~~~~~~~ Set up an initial profile at ``/etc/salt/cloud.profiles``: .. code-block:: yaml diff --git a/salt/cloud/clouds/softlayer.py b/salt/cloud/clouds/softlayer.py index d24bcab660..95bf6c10c0 100644 --- a/salt/cloud/clouds/softlayer.py +++ b/salt/cloud/clouds/softlayer.py @@ -371,6 +371,12 @@ def create(vm_): if post_uri: kwargs['postInstallScriptUri'] = post_uri + dedicated_host_id = config.get_cloud_config_value( + 'dedicated_host_id', vm_, __opts__, default=None + ) + if dedicated_host_id: + kwargs['dedicatedHost'] = {'id': dedicated_host_id} + __utils__['cloud.fire_event']( 'event', 'requesting instance', From 8a1c575ae52fcffd057a9de3c7db1b0a0225ac35 Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 25 Oct 2017 09:59:04 -0400 Subject: [PATCH 175/184] Update salt.utils.fopen calls to new salt.utils.files.fopen path --- tests/support/case.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/support/case.py b/tests/support/case.py index b69eb2656f..62f7e078d7 100644 --- a/tests/support/case.py +++ b/tests/support/case.py @@ -599,16 +599,17 @@ class SPMCase(TestCase, AdaptedConfigurationTestCaseMixin): for f_dir in dirs: os.makedirs(f_dir) - import salt.utils + # Late import + import salt.utils.files - with salt.utils.fopen(self.formula_sls, 'w') as fp: + with salt.utils.files.fopen(self.formula_sls, 'w') as fp: fp.write(textwrap.dedent('''\ install-apache: pkg.installed: - name: apache2 ''')) - with salt.utils.fopen(self.formula_file, 'w') as fp: + with salt.utils.files.fopen(self.formula_file, 'w') as fp: fp.write(textwrap.dedent('''\ name: apache os: RedHat, Debian, Ubuntu, Suse, FreeBSD @@ -655,9 +656,10 @@ class SPMCase(TestCase, AdaptedConfigurationTestCaseMixin): repo_conf_dir = self.config['spm_repos_config'] + '.d' os.makedirs(repo_conf_dir) - import salt.utils + # Late import + import salt.utils.files - with salt.utils.fopen(os.path.join(repo_conf_dir, 'spm.repo'), 'w') as fp: + with salt.utils.files.fopen(os.path.join(repo_conf_dir, 'spm.repo'), 'w') as fp: fp.write(textwrap.dedent('''\ local_repo: url: file://{0} From 5e2cab0d5e663af58232624443d98fc741852fe4 Mon Sep 17 00:00:00 2001 From: Jeffrey 'jf' Lim Date: Thu, 26 Oct 2017 02:08:16 +0800 Subject: [PATCH 176/184] Fix utils.files.guess_archive_type to recognize the "tbz" extension as well (also tidy up list of extensions) --- salt/utils/files.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/utils/files.py b/salt/utils/files.py index 64c7a55878..d6db7cfbc1 100644 --- a/salt/utils/files.py +++ b/salt/utils/files.py @@ -59,7 +59,9 @@ def guess_archive_type(name): Guess an archive type (tar, zip, or rar) by its file extension ''' name = name.lower() - for ending in ('tar', 'tar.gz', 'tar.bz2', 'tar.xz', 'tgz', 'tbz2', 'txz', + for ending in ('tar', 'tar.gz', 'tgz', + 'tar.bz2', 'tbz2', 'tbz', + 'tar.xz', 'txz', 'tar.lzma', 'tlz'): if name.endswith('.' + ending): return 'tar' From 35c6a026896e0fe272174174eff53e120a5900b1 Mon Sep 17 00:00:00 2001 From: Nasenbaer Date: Thu, 26 Oct 2017 10:29:43 +0200 Subject: [PATCH 177/184] Add sentry config deprecation to release notes --- doc/topics/releases/oxygen.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/topics/releases/oxygen.rst b/doc/topics/releases/oxygen.rst index d6a357d732..023384f366 100644 --- a/doc/topics/releases/oxygen.rst +++ b/doc/topics/releases/oxygen.rst @@ -1095,3 +1095,10 @@ The ``version.py`` file had the following changes: Warnings for moving away from the ``env`` option were removed. ``saltenv`` should be used instead. The removal of these warnings does not have a behavior change. Only the warning text was removed. + +Sentry Log Handler +------------------ + +Configuring sentry raven python client via ``project``, ``servers``, ``public_key +and ``secret_key`` is deprecated and won't work with sentry clients > 3.0. +Instead, the ``dsn`` config param must be used. From 412506e6268e31d8fd837d8acdae915f58aba66c Mon Sep 17 00:00:00 2001 From: Jeffrey 'jf' Lim Date: Thu, 26 Oct 2017 23:22:05 +0800 Subject: [PATCH 178/184] Add `prepend_path` option to `cmd.run` (meant as a response to #21840) --- salt/modules/cmdmod.py | 34 ++++++++++++++++++++++++++++++++++ salt/states/cmd.py | 6 ++++++ 2 files changed, 40 insertions(+) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index c3c2070126..d2930028a8 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -269,6 +269,7 @@ def _run(cmd, python_shell=False, env=None, clean_env=False, + prepend_path=None, rstrip=True, template=None, umask=None, @@ -492,6 +493,9 @@ def _run(cmd, run_env = os.environ.copy() run_env.update(env) + if prepend_path: + run_env['PATH'] = prepend_path + ':' + run_env['PATH'] + if python_shell is None: python_shell = False @@ -768,6 +772,7 @@ def run(cmd, python_shell=None, env=None, clean_env=False, + prepend_path=None, template=None, rstrip=True, umask=None, @@ -864,6 +869,9 @@ def run(cmd, variables and set only those provided in the 'env' argument to this function. + :param str prepend_path: $PATH segment to prepend (trailing ':' not necessary) + to $PATH + :param str template: If this setting is applied then the named templating engine will be used to render the downloaded file. Currently jinja, mako, and wempy are supported @@ -949,6 +957,7 @@ def run(cmd, stderr=subprocess.STDOUT, env=env, clean_env=clean_env, + prepend_path=prepend_path, template=template, rstrip=rstrip, umask=umask, @@ -991,6 +1000,7 @@ def shell(cmd, shell=DEFAULT_SHELL, env=None, clean_env=False, + prepend_path=None, template=None, rstrip=True, umask=None, @@ -1079,6 +1089,9 @@ def shell(cmd, variables and set only those provided in the 'env' argument to this function. + :param str prepend_path: $PATH segment to prepend (trailing ':' not necessary) + to $PATH + :param str template: If this setting is applied then the named templating engine will be used to render the downloaded file. Currently jinja, mako, and wempy are supported @@ -1157,6 +1170,7 @@ def shell(cmd, shell=shell, env=env, clean_env=clean_env, + prepend_path=prepend_path, template=template, rstrip=rstrip, umask=umask, @@ -1182,6 +1196,7 @@ def run_stdout(cmd, python_shell=None, env=None, clean_env=False, + prepend_path=None, template=None, rstrip=True, umask=None, @@ -1265,6 +1280,9 @@ def run_stdout(cmd, variables and set only those provided in the 'env' argument to this function. + :param str prepend_path: $PATH segment to prepend (trailing ':' not necessary) + to $PATH + :param str template: If this setting is applied then the named templating engine will be used to render the downloaded file. Currently jinja, mako, and wempy are supported @@ -1319,6 +1337,7 @@ def run_stdout(cmd, python_shell=python_shell, env=env, clean_env=clean_env, + prepend_path=prepend_path, template=template, rstrip=rstrip, umask=umask, @@ -1363,6 +1382,7 @@ def run_stderr(cmd, python_shell=None, env=None, clean_env=False, + prepend_path=None, template=None, rstrip=True, umask=None, @@ -1447,6 +1467,9 @@ def run_stderr(cmd, variables and set only those provided in the 'env' argument to this function. + :param str prepend_path: $PATH segment to prepend (trailing ':' not necessary) + to $PATH + :param str template: If this setting is applied then the named templating engine will be used to render the downloaded file. Currently jinja, mako, and wempy are supported @@ -1501,6 +1524,7 @@ def run_stderr(cmd, python_shell=python_shell, env=env, clean_env=clean_env, + prepend_path=prepend_path, template=template, rstrip=rstrip, umask=umask, @@ -1545,6 +1569,7 @@ def run_all(cmd, python_shell=None, env=None, clean_env=False, + prepend_path=None, template=None, rstrip=True, umask=None, @@ -1631,6 +1656,9 @@ def run_all(cmd, variables and set only those provided in the 'env' argument to this function. + :param str prepend_path: $PATH segment to prepend (trailing ':' not necessary) + to $PATH + :param str template: If this setting is applied then the named templating engine will be used to render the downloaded file. Currently jinja, mako, and wempy are supported @@ -1709,6 +1737,7 @@ def run_all(cmd, python_shell=python_shell, env=env, clean_env=clean_env, + prepend_path=prepend_path, template=template, rstrip=rstrip, umask=umask, @@ -3458,6 +3487,7 @@ def run_bg(cmd, python_shell=None, env=None, clean_env=False, + prepend_path=None, template=None, umask=None, timeout=None, @@ -3545,6 +3575,9 @@ def run_bg(cmd, variables and set only those provided in the 'env' argument to this function. + :param str prepend_path: $PATH segment to prepend (trailing ':' not necessary) + to $PATH + :param str template: If this setting is applied then the named templating engine will be used to render the downloaded file. Currently jinja, mako, and wempy are supported @@ -3613,6 +3646,7 @@ def run_bg(cmd, cwd=cwd, env=env, clean_env=clean_env, + prepend_path=prepend_path, template=template, umask=umask, log_callback=log_callback, diff --git a/salt/states/cmd.py b/salt/states/cmd.py index 3d5a13b959..dbc245ab19 100644 --- a/salt/states/cmd.py +++ b/salt/states/cmd.py @@ -638,6 +638,7 @@ def run(name, runas=None, shell=None, env=None, + prepend_path=None, stateful=False, umask=None, output_loglevel='debug', @@ -712,6 +713,10 @@ def run(name, - env: - PATH: {{ [current_path, '/my/special/bin']|join(':') }} + prepend_path + $PATH segment to prepend (trailing ':' not necessary) to $PATH. This is + an easier alternative to the Jinja workaround. + stateful The command being executed is expected to return data about executing a state. For more information, see the :ref:`stateful-argument` section. @@ -807,6 +812,7 @@ def run(name, 'use_vt': use_vt, 'shell': shell or __grains__['shell'], 'env': env, + 'prepend_path': prepend_path, 'umask': umask, 'output_loglevel': output_loglevel, 'quiet': quiet}) From b3b6c6fc3a60b41a88caaf6da52bb911d417c24f Mon Sep 17 00:00:00 2001 From: Jeffrey 'jf' Lim Date: Thu, 26 Oct 2017 23:23:09 +0800 Subject: [PATCH 179/184] Fix pydoc indentation for `cmdmod.run_chroot`, `clean_env` option --- salt/modules/cmdmod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index d2930028a8..65cf6517b2 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -2602,7 +2602,7 @@ def run_chroot(root, - env: - PATH: {{ [current_path, '/my/special/bin']|join(':') }} - clean_env: + clean_env: Attempt to clean out all other shell environment variables and set only those provided in the 'env' argument to this function. From 09d0a347c51597089446140b1ccea5aa89c264a8 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 27 Oct 2017 09:48:08 -0400 Subject: [PATCH 180/184] Reduce the number of days an issue is stale by 15 --- .github/stale.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/stale.yml b/.github/stale.yml index 754018e841..798d9d32b2 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -1,8 +1,8 @@ # Probot Stale configuration file # Number of days of inactivity before an issue becomes stale -# 925 is approximately 2 years and 6 months -daysUntilStale: 925 +# 910 is approximately 2 years and 6 months +daysUntilStale: 910 # Number of days of inactivity before a stale issue is closed daysUntilClose: 7 From bbdcaee25028f34f202b1ad5b5b04f77739e01d3 Mon Sep 17 00:00:00 2001 From: Jeffrey 'jf' Lim Date: Fri, 27 Oct 2017 23:37:06 +0800 Subject: [PATCH 181/184] tag `prepend_path` option with '.. versionadded:: Oxygen' --- salt/modules/cmdmod.py | 12 ++++++++++++ salt/states/cmd.py | 2 ++ 2 files changed, 14 insertions(+) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index 65cf6517b2..30cb08dd8b 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -872,6 +872,8 @@ def run(cmd, :param str prepend_path: $PATH segment to prepend (trailing ':' not necessary) to $PATH + .. versionadded:: Oxygen + :param str template: If this setting is applied then the named templating engine will be used to render the downloaded file. Currently jinja, mako, and wempy are supported @@ -1092,6 +1094,8 @@ def shell(cmd, :param str prepend_path: $PATH segment to prepend (trailing ':' not necessary) to $PATH + .. versionadded:: Oxygen + :param str template: If this setting is applied then the named templating engine will be used to render the downloaded file. Currently jinja, mako, and wempy are supported @@ -1283,6 +1287,8 @@ def run_stdout(cmd, :param str prepend_path: $PATH segment to prepend (trailing ':' not necessary) to $PATH + .. versionadded:: Oxygen + :param str template: If this setting is applied then the named templating engine will be used to render the downloaded file. Currently jinja, mako, and wempy are supported @@ -1470,6 +1476,8 @@ def run_stderr(cmd, :param str prepend_path: $PATH segment to prepend (trailing ':' not necessary) to $PATH + .. versionadded:: Oxygen + :param str template: If this setting is applied then the named templating engine will be used to render the downloaded file. Currently jinja, mako, and wempy are supported @@ -1659,6 +1667,8 @@ def run_all(cmd, :param str prepend_path: $PATH segment to prepend (trailing ':' not necessary) to $PATH + .. versionadded:: Oxygen + :param str template: If this setting is applied then the named templating engine will be used to render the downloaded file. Currently jinja, mako, and wempy are supported @@ -3578,6 +3588,8 @@ def run_bg(cmd, :param str prepend_path: $PATH segment to prepend (trailing ':' not necessary) to $PATH + .. versionadded:: Oxygen + :param str template: If this setting is applied then the named templating engine will be used to render the downloaded file. Currently jinja, mako, and wempy are supported diff --git a/salt/states/cmd.py b/salt/states/cmd.py index dbc245ab19..f2f2c60a82 100644 --- a/salt/states/cmd.py +++ b/salt/states/cmd.py @@ -717,6 +717,8 @@ def run(name, $PATH segment to prepend (trailing ':' not necessary) to $PATH. This is an easier alternative to the Jinja workaround. + .. versionadded:: Oxygen + stateful The command being executed is expected to return data about executing a state. For more information, see the :ref:`stateful-argument` section. From 6bd8aea8cfa5722ef78c0a16cd4ab95bccf25d98 Mon Sep 17 00:00:00 2001 From: Jeffrey 'jf' Lim Date: Sat, 28 Oct 2017 00:02:12 +0800 Subject: [PATCH 182/184] cmdmod: shift new `prepend_path` argument to the end to avoid breaking any positional argument calls --- salt/modules/cmdmod.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index 30cb08dd8b..766d0f99b5 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -772,7 +772,6 @@ def run(cmd, python_shell=None, env=None, clean_env=False, - prepend_path=None, template=None, rstrip=True, umask=None, @@ -787,6 +786,7 @@ def run(cmd, password=None, encoded_cmd=False, raise_err=False, + prepend_path=None, **kwargs): r''' Execute the passed command and return the output as a string @@ -1002,7 +1002,6 @@ def shell(cmd, shell=DEFAULT_SHELL, env=None, clean_env=False, - prepend_path=None, template=None, rstrip=True, umask=None, @@ -1016,6 +1015,7 @@ def shell(cmd, use_vt=False, bg=False, password=None, + prepend_path=None, **kwargs): ''' Execute the passed command and return the output as a string. @@ -1200,7 +1200,6 @@ def run_stdout(cmd, python_shell=None, env=None, clean_env=False, - prepend_path=None, template=None, rstrip=True, umask=None, @@ -1212,6 +1211,7 @@ def run_stdout(cmd, saltenv='base', use_vt=False, password=None, + prepend_path=None, **kwargs): ''' Execute a command, and only return the standard out @@ -1388,7 +1388,6 @@ def run_stderr(cmd, python_shell=None, env=None, clean_env=False, - prepend_path=None, template=None, rstrip=True, umask=None, @@ -1400,6 +1399,7 @@ def run_stderr(cmd, saltenv='base', use_vt=False, password=None, + prepend_path=None, **kwargs): ''' Execute a command and only return the standard error @@ -1577,7 +1577,6 @@ def run_all(cmd, python_shell=None, env=None, clean_env=False, - prepend_path=None, template=None, rstrip=True, umask=None, @@ -1591,6 +1590,7 @@ def run_all(cmd, redirect_stderr=False, password=None, encoded_cmd=False, + prepend_path=None, **kwargs): ''' Execute the passed command and return a dict of return data @@ -3497,7 +3497,6 @@ def run_bg(cmd, python_shell=None, env=None, clean_env=False, - prepend_path=None, template=None, umask=None, timeout=None, @@ -3507,6 +3506,7 @@ def run_bg(cmd, ignore_retcode=False, saltenv='base', password=None, + prepend_path=None, **kwargs): r''' .. versionadded: 2016.3.0 From f59271f17268915c1b032c904ecfd3b7d208e68f Mon Sep 17 00:00:00 2001 From: Jeffrey 'jf' Lim Date: Sat, 28 Oct 2017 00:18:29 +0800 Subject: [PATCH 183/184] tweak prepend_path join code --- salt/modules/cmdmod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index 766d0f99b5..3ae58905c3 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -494,7 +494,7 @@ def _run(cmd, run_env.update(env) if prepend_path: - run_env['PATH'] = prepend_path + ':' + run_env['PATH'] + run_env['PATH'] = ':'.join((prepend_path, run_env['PATH'])) if python_shell is None: python_shell = False From b92d303756294fc0e989692453d334a7f53eaf8b Mon Sep 17 00:00:00 2001 From: Zhi Han Date: Fri, 27 Oct 2017 15:39:28 -0400 Subject: [PATCH 184/184] Typo, close the parenthesis. --- salt/states/host.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/host.py b/salt/states/host.py index 8345168cd5..a07cf18ffb 100644 --- a/salt/states/host.py +++ b/salt/states/host.py @@ -131,7 +131,7 @@ def absent(name, ip): # pylint: disable=C0103 comments.append('Host {0} ({1}) already absent'.format(name, _ip)) else: if __opts__['test']: - comments.append('Host {0} ({1} needs to be removed'.format(name, _ip)) + comments.append('Host {0} ({1}) needs to be removed'.format(name, _ip)) else: if __salt__['hosts.rm_host'](_ip, name): ret['changes'] = {'host': name}