diff --git a/.mention-bot b/.mention-bot new file mode 100644 index 0000000000..c3181cdc9f --- /dev/null +++ b/.mention-bot @@ -0,0 +1,7 @@ +{ + "skipTitle": "Merge forward", + "delayed": true, + "delayedUntil": "2h", + "userBlacklist": [] +} + diff --git a/conf/minion b/conf/minion index 62e3738293..5a0d5f4baa 100644 --- a/conf/minion +++ b/conf/minion @@ -649,6 +649,10 @@ ########################################### # Disable multiprocessing support, by default when a minion receives a # publication a new process is spawned and the command is executed therein. +# +# WARNING: Disabling multiprocessing may result in substantial slowdowns +# when processing large pillars. See https://github.com/saltstack/salt/issues/38758 +# for a full explanation. #multiprocessing: True diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index d43581a6f8..671801650c 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -81,9 +81,10 @@ The option can also be set to a list of masters, enabling ``ipv6`` -------- -Default: ``False`` +Default: ``None`` -Whether the master should be connected over IPv6. +Whether the master should be connected over IPv6. By default salt minion +will try to automatically detect IPv6 connectivity to master. .. code-block:: yaml diff --git a/doc/topics/development/contributing.rst b/doc/topics/development/contributing.rst index 2c0afa048b..57e3b3c677 100644 --- a/doc/topics/development/contributing.rst +++ b/doc/topics/development/contributing.rst @@ -419,3 +419,14 @@ and bug resolution. See the :ref:`Labels and Milestones .. _`Closing issues via commit message`: https://help.github.com/articles/closing-issues-via-commit-messages .. _`git format-patch`: https://www.kernel.org/pub/software/scm/git/docs/git-format-patch.html .. _salt-users: https://groups.google.com/forum/#!forum/salt-users + +Mentionbot +========== + +SaltStack runs a mention-bot which notifies contributors who might be able +to help review incoming pull-requests based on their past contribution to +files which are being changed. + +If you do not wish to receive these notifications, please add your GitHub +handle to the blacklist line in the `.mention-bot` file located in the +root of the Salt repository. diff --git a/doc/topics/releases/2016.11.3.rst b/doc/topics/releases/2016.11.3.rst index 2b285a8883..2adeda6ac9 100644 --- a/doc/topics/releases/2016.11.3.rst +++ b/doc/topics/releases/2016.11.3.rst @@ -10,17 +10,152 @@ Changes for v2016.11.2..v2016.11.3 Extended changelog courtesy of Todd Stansell (https://github.com/tjstansell/salt-changelogs): -*Generated at: 2017-02-16T16:48:51Z* +*Generated at: 2017-02-22T23:01:16Z* Statistics: -- Total Merges: **126** -- Total Issue references: **75** -- Total PR references: **199** +- Total Merges: **139** +- Total Issue references: **78** +- Total PR references: **217** Changes: +- **PR** `#39536`_: (*twangboy*) Namespace 'status' functions in 'win_status' + @ *2017-02-21T23:45:31Z* + + - **PR** `#39005`_: (*cro*) Ungate the status.py module and raise unsupported errors in functions not executable on Windows. + | refs: `#39536`_ + * 40f72db Merge pull request `#39536`_ from twangboy/fix_win_status + * d5453e2 Remove unused import (lint) + + * 837c32e Remove list2cmdline + + * c258cb3 Streamline wmic command returns for easier parsing + + * 6d2cf81 Fix 'ping_master' function + + * d946d10 Namespace 'status' functions in 'win_status' + +- **PR** `#39534`_: (*rallytime*) Fix breakage in aptpkg and dpkg execution modules + @ *2017-02-21T20:31:15Z* + + - **PR** `#39418`_: (*anlutro*) Allow aptpkg.info_installed on package names that aren't installed + | refs: `#39534`_ + * dc8f578 Merge pull request `#39534`_ from rallytime/fix-pkg-function-specs + * d34a8fe Fix breakage in aptpkg and dpkg execution modules + +* 1d0d7b2 Upgrade SaltTesting to run test suite for 2016.11 and add SaltPyLint (`#39521`_) + + - **ISSUE** `#34712`_: (*richardscollin*) Salt Test Suite Error - develop + | refs: `#37366`_ + - **PR** `#39521`_: (*vutny*) Upgrade SaltTesting to run test suite for 2016.11 and add SaltPyLint + - **PR** `#37366`_: (*eradman*) dev_python*.txt: use current SaltTesting and SaltPyLint modules + | refs: `#39521`_ + +- **PR** `#39370`_: (*twangboy*) Gate win_osinfo and winservice + @ *2017-02-17T23:53:58Z* + + * e4c7168 Merge pull request `#39370`_ from twangboy/gate_win_utils + * 167cdb3 Gate windows specific imports, add __virtual__ + + * e67387d Add option to return a Non instantiated class + + * 315b0cc Clarify return value for win_osinfo + + * 994314e Fix more docs + + * 2bbe3cb Fix some docs + + * 4103563 Merge branch 'gate_win_utils' of https://github.com/twangboy/salt into gate_win_utils + + * 24c1bd0 Remove extra newlines + + * 82a86ce Add helper function for winservice + + * 0051b5a Put the win_osinfo classes in a helper function + + * 4e08534 Gate win_osinfo and winservice better + +- **PR** `#39486`_: (*twangboy*) Remove orphaned function list_configurable_policies + @ *2017-02-17T22:21:50Z* + + * a3e71b6 Merge pull request `#39486`_ from twangboy/win_remove_orphaned + * 1328055 Remove orphaned function list_configurable_policies + +- **PR** `#39418`_: (*anlutro*) Allow aptpkg.info_installed on package names that aren't installed + | refs: `#39534`_ + @ *2017-02-17T18:34:19Z* + + * 87b269f Merge pull request `#39418`_ from alprs/fix-aptpkg_info_nonexistent_pkg + * 246bf1e add failhard argument to various apt pkg functions + +- **PR** `#39438`_: (*mirceaulinic*) file.get_managed: refetch source when file hashsum is changed + @ *2017-02-17T17:58:29Z* + + * e816d6c Merge pull request `#39438`_ from cloudflare/fix_39422 + * 8453800 file.get_managed: refetch cached file when hashsum chnaged + +- **PR** `#39432`_: (*dmaziuk*) Quick and dirty fix for GECOS fields with more than 3 commas + @ *2017-02-17T17:57:30Z* + + - **ISSUE** `#39203`_: (*dmaziuk*) salt.users gecos field + | refs: `#39432`_ `#39432`_ + * a5fe8f0 Merge pull request `#39432`_ from dmaziuk/issue39203 + * 41c0463 Remove # + + * 4f877c6 Quick and dirty fix for GECOS fields with more than 3 commas + +- **PR** `#39484`_: (*corywright*) The Reactor docs should use pillar='{}' instead of 'pillar={}' + @ *2017-02-17T17:50:57Z* + + * 3665229 Merge pull request `#39484`_ from corywright/fix-reactor-docs-pillar-keyword-args + * cc90d0d The Reactor docs should use pillar='{}' instead of 'pillar={}' + +- **PR** `#39456`_: (*twangboy*) Add salt icon to buildenv directory + @ *2017-02-16T22:47:58Z* + + * 2e3a9c5 Merge pull request `#39456`_ from twangboy/win_fix_icon + * 8dd915d Add salt icon to buildenv directory + +- **PR** `#39462`_: (*twangboy*) Use url_path instead of url_data.path + @ *2017-02-16T22:44:18Z* + + * 63adc03 Merge pull request `#39462`_ from twangboy/win_fix_fileclient + * a96bc13 Use url_path instead of url_data.path + +- **PR** `#39458`_: (*rallytime*) Fix more warnings in doc build + @ *2017-02-16T21:45:52Z* + + * e9b034f Merge pull request `#39458`_ from rallytime/fixup-more-doc-build-warnings + * e698bc3 Fix more warnings in doc build + +- **PR** `#39437`_: (*sakateka*) Fixes about saltfile + @ *2017-02-16T20:32:15Z* + + * e4f8c2b Merge pull request `#39437`_ from sakateka/fixes_about_saltfile + * ab68524 less pylint: salt/utils/parsers.py + + * 9e7d9dc Revert "pylint: salt/utils/parsers.py" + + * f3f129c document ~/.salt/Saltfile + + * 33f3614 pylint: salt/utils/parsers.py + + * 0f36e10 expand config_dir and '~/.salt/Saltfile' as last resort + +* 1acf00d add 2016.11.3 changelog to release notes (`#39451`_) + + - **PR** `#39451`_: (*Ch3LL*) add 2016.11.3 changelog to release notes + +- **PR** `#39448`_: (*gtmanfred*) Add release notes for cisco proxy minions added in Carbon + @ *2017-02-16T17:29:48Z* + + - **ISSUE** `#38032`_: (*meggiebot*) Add missing Carbon docs + | refs: `#39448`_ + * 8e2cbd2 Merge pull request `#39448`_ from gtmanfred/2016.11 + * 3172e88 Add release notes for cisco proxy minions added in Carbon + - **PR** `#39428`_: (*rallytime*) [2016.11] Merge forward from 2016.3 to 2016.11 @ *2017-02-16T00:01:15Z* @@ -846,6 +981,7 @@ Changes: * a7fc02e Ungate the status.py module and raise unsupported errors in functions not executeable on Windows. (`#39005`_) - **PR** `#39005`_: (*cro*) Ungate the status.py module and raise unsupported errors in functions not executable on Windows. + | refs: `#39536`_ - **PR** `#39012`_: (*terminalmage*) Fix "invalid lexer" errors in docs build @ *2017-01-28T06:47:45Z* @@ -1252,6 +1388,7 @@ Changes: .. _`#33890`: https://github.com/saltstack/salt/issues/33890 .. _`#34280`: https://github.com/saltstack/salt/pull/34280 .. _`#34551`: https://github.com/saltstack/salt/issues/34551 +.. _`#34712`: https://github.com/saltstack/salt/issues/34712 .. _`#34780`: https://github.com/saltstack/salt/issues/34780 .. _`#35055`: https://github.com/saltstack/salt/pull/35055 .. _`#35777`: https://github.com/saltstack/salt/issues/35777 @@ -1264,12 +1401,14 @@ Changes: .. _`#37174`: https://github.com/saltstack/salt/issues/37174 .. _`#37262`: https://github.com/saltstack/salt/pull/37262 .. _`#37338`: https://github.com/saltstack/salt/pull/37338 +.. _`#37366`: https://github.com/saltstack/salt/pull/37366 .. _`#37375`: https://github.com/saltstack/salt/pull/37375 .. _`#37413`: https://github.com/saltstack/salt/issues/37413 .. _`#37632`: https://github.com/saltstack/salt/pull/37632 .. _`#37864`: https://github.com/saltstack/salt/pull/37864 .. _`#37938`: https://github.com/saltstack/salt/issues/37938 .. _`#38003`: https://github.com/saltstack/salt/issues/38003 +.. _`#38032`: https://github.com/saltstack/salt/issues/38032 .. _`#38081`: https://github.com/saltstack/salt/issues/38081 .. _`#38100`: https://github.com/saltstack/salt/issues/38100 .. _`#38165`: https://github.com/saltstack/salt/pull/38165 @@ -1435,6 +1574,7 @@ Changes: .. _`#39198`: https://github.com/saltstack/salt/pull/39198 .. _`#39199`: https://github.com/saltstack/salt/pull/39199 .. _`#39202`: https://github.com/saltstack/salt/pull/39202 +.. _`#39203`: https://github.com/saltstack/salt/issues/39203 .. _`#39206`: https://github.com/saltstack/salt/pull/39206 .. _`#39209`: https://github.com/saltstack/salt/pull/39209 .. _`#39210`: https://github.com/saltstack/salt/pull/39210 @@ -1484,16 +1624,31 @@ Changes: .. _`#39362`: https://github.com/saltstack/salt/pull/39362 .. _`#39364`: https://github.com/saltstack/salt/pull/39364 .. _`#39369`: https://github.com/saltstack/salt/pull/39369 +.. _`#39370`: https://github.com/saltstack/salt/pull/39370 .. _`#39378`: https://github.com/saltstack/salt/pull/39378 .. _`#39379`: https://github.com/saltstack/salt/pull/39379 .. _`#39380`: https://github.com/saltstack/salt/pull/39380 .. _`#39392`: https://github.com/saltstack/salt/pull/39392 .. _`#39400`: https://github.com/saltstack/salt/pull/39400 .. _`#39409`: https://github.com/saltstack/salt/pull/39409 +.. _`#39418`: https://github.com/saltstack/salt/pull/39418 .. _`#39419`: https://github.com/saltstack/salt/pull/39419 .. _`#39424`: https://github.com/saltstack/salt/pull/39424 .. _`#39428`: https://github.com/saltstack/salt/pull/39428 .. _`#39429`: https://github.com/saltstack/salt/pull/39429 +.. _`#39432`: https://github.com/saltstack/salt/pull/39432 +.. _`#39437`: https://github.com/saltstack/salt/pull/39437 +.. _`#39438`: https://github.com/saltstack/salt/pull/39438 +.. _`#39448`: https://github.com/saltstack/salt/pull/39448 +.. _`#39451`: https://github.com/saltstack/salt/pull/39451 +.. _`#39456`: https://github.com/saltstack/salt/pull/39456 +.. _`#39458`: https://github.com/saltstack/salt/pull/39458 +.. _`#39462`: https://github.com/saltstack/salt/pull/39462 +.. _`#39484`: https://github.com/saltstack/salt/pull/39484 +.. _`#39486`: https://github.com/saltstack/salt/pull/39486 +.. _`#39521`: https://github.com/saltstack/salt/pull/39521 +.. _`#39534`: https://github.com/saltstack/salt/pull/39534 +.. _`#39536`: https://github.com/saltstack/salt/pull/39536 .. _`bp-36336`: https://github.com/saltstack/salt/pull/36336 .. _`bp-37338`: https://github.com/saltstack/salt/pull/37338 .. _`bp-37375`: https://github.com/saltstack/salt/pull/37375 diff --git a/doc/topics/transports/tcp.rst b/doc/topics/transports/tcp.rst index 1707ec3f32..d12bde129e 100644 --- a/doc/topics/transports/tcp.rst +++ b/doc/topics/transports/tcp.rst @@ -57,8 +57,13 @@ The minimal `ssl` option in the minion configuration file looks like this: .. code-block:: yaml + ssl: True + # Versions below 2016.11.4: ssl: {} +Specific options can be sent to the minion also, as defined in the Python +`ssl.wrap_socket` function. + .. note:: While setting the ssl_version is not required, we recomend it. Some older diff --git a/salt/beacons/avahi_announce.py b/salt/beacons/avahi_announce.py index dab81c1953..117e1ddd23 100644 --- a/salt/beacons/avahi_announce.py +++ b/salt/beacons/avahi_announce.py @@ -67,6 +67,26 @@ def __validate__(config): return True, 'Valid beacon configuration' +def _enforce_txt_record_maxlen(key, value): + ''' + Enforces the TXT record maximum length of 255 characters. + TXT record length includes key, value, and '='. + + :param str key: Key of the TXT record + :param str value: Value of the TXT record + + :rtype: str + :return: The value of the TXT record. It may be truncated if it exceeds + the maximum permitted length. In case of truncation, '...' is + appended to indicate that the entire value is not present. + ''' + # Add 1 for '=' seperator between key and value + if len(key) + len(value) + 1 > 255: + # 255 - 3 ('...') - 1 ('=') = 251 + return value[:251 - len(key)] + '...' + return value + + def beacon(config): ''' Broadcast values via zeroconf @@ -158,11 +178,11 @@ def beacon(config): grain_value = grain_value[grain_index] else: grain_value = ','.join(grain_value) - txt[item] = grain_value + txt[item] = _enforce_txt_record_maxlen(item, grain_value) if LAST_GRAINS and (LAST_GRAINS.get(grain, '') != __grains__.get(grain, '')): changes[str('txt.' + item)] = txt[item] else: - txt[item] = config['txt'][item] + txt[item] = _enforce_txt_record_maxlen(item, config['txt'][item]) if not LAST_GRAINS: changes[str('txt.' + item)] = txt[item] diff --git a/salt/beacons/bonjour_announce.py b/salt/beacons/bonjour_announce.py index 5cff20cc14..0af2928928 100644 --- a/salt/beacons/bonjour_announce.py +++ b/salt/beacons/bonjour_announce.py @@ -60,6 +60,26 @@ def __validate__(config): return True, 'Valid beacon configuration' +def _enforce_txt_record_maxlen(key, value): + ''' + Enforces the TXT record maximum length of 255 characters. + TXT record length includes key, value, and '='. + + :param str key: Key of the TXT record + :param str value: Value of the TXT record + + :rtype: str + :return: The value of the TXT record. It may be truncated if it exceeds + the maximum permitted length. In case of truncation, '...' is + appended to indicate that the entire value is not present. + ''' + # Add 1 for '=' seperator between key and value + if len(key) + len(value) + 1 > 255: + # 255 - 3 ('...') - 1 ('=') = 251 + return value[:251 - len(key)] + '...' + return value + + def beacon(config): ''' Broadcast values via zeroconf @@ -152,11 +172,11 @@ def beacon(config): grain_value = grain_value[grain_index] else: grain_value = ','.join(grain_value) - txt[item] = grain_value + txt[item] = _enforce_txt_record_maxlen(item, grain_value) if LAST_GRAINS and (LAST_GRAINS.get(grain, '') != __grains__.get(grain, '')): changes[str('txt.' + item)] = txt[item] else: - txt[item] = config['txt'][item] + txt[item] = _enforce_txt_record_maxlen(item, config['txt'][item]) if not LAST_GRAINS: changes[str('txt.' + item)] = txt[item] diff --git a/salt/cli/batch.py b/salt/cli/batch.py index 579983ba7b..f18fbeb586 100644 --- a/salt/cli/batch.py +++ b/salt/cli/batch.py @@ -58,21 +58,21 @@ class Batch(object): # Broadcast to targets fret = set() nret = set() - try: - for ret in ping_gen: - if ('minions' and 'jid') in ret: - for minion in ret['minions']: - nret.add(minion) - continue - else: + for ret in ping_gen: + if ('minions' and 'jid') in ret: + for minion in ret['minions']: + nret.add(minion) + continue + else: + try: m = next(six.iterkeys(ret)) - if m is not None: - fret.add(m) - return (list(fret), ping_gen, nret.difference(fret)) - except StopIteration: - if not self.quiet: - print_cli('No minions matched the target.') - return list(fret), ping_gen + except StopIteration: + if not self.quiet: + print_cli('No minions matched the target.') + break + if m is not None: + fret.add(m) + return (list(fret), ping_gen, nret.difference(fret)) def get_bnum(self): ''' diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 703917d5ac..97eb4aeb00 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -1139,7 +1139,7 @@ DEFAULT_MINION_OPTS = { 'mine_interval': 60, 'ipc_mode': _DFLT_IPC_MODE, 'ipc_write_buffer': _DFLT_IPC_WBUFFER, - 'ipv6': False, + 'ipv6': None, 'file_buffer_size': 262144, 'tcp_pub_port': 4510, 'tcp_pull_port': 4511, diff --git a/salt/fileclient.py b/salt/fileclient.py index 6ed8723f4f..c55716bced 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -1340,8 +1340,7 @@ class FSClient(RemoteClient): the FSChan object ''' def __init__(self, opts): # pylint: disable=W0231 - self.opts = opts - self.utils = salt.loader.utils(opts) + Client.__init__(self, opts) # pylint: disable=W0233 self.channel = salt.fileserver.FSChan(opts) self.auth = DumbAuth() diff --git a/salt/loader.py b/salt/loader.py index cb12cd9564..5eb5e8cb0f 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -1046,7 +1046,8 @@ class LazyLoader(salt.utils.lazy.LazyDict): self.pack = {} if pack is None else pack if opts is None: opts = {} - self.context_dict = salt.utils.context.ContextDict() + threadsafety = not opts.get('multiprocessing') + self.context_dict = salt.utils.context.ContextDict(threadsafe=threadsafety) self.opts = self.__prep_mod_opts(opts) self.module_dirs = module_dirs diff --git a/salt/minion.py b/salt/minion.py index 66f2bbdf6d..266af51dac 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -133,7 +133,7 @@ log = logging.getLogger(__name__) # 6. Handle publications -def resolve_dns(opts, fallback=True): +def resolve_dns(opts, fallback=True, connect=True): ''' Resolves the master_ip and master_uri options ''' @@ -150,13 +150,13 @@ def resolve_dns(opts, fallback=True): if opts['master'] == '': raise SaltSystemExit ret['master_ip'] = \ - salt.utils.dns_check(opts['master'], True, opts['ipv6']) + salt.utils.dns_check(opts['master'], opts['master_port'], True, opts['ipv6'], connect) except SaltClientError: if opts['retry_dns']: while True: import salt.log - msg = ('Master hostname: \'{0}\' not found. Retrying in {1} ' - 'seconds').format(opts['master'], opts['retry_dns']) + msg = ('Master hostname: \'{0}\' not found or not responsive. ' + 'Retrying in {1} seconds').format(opts['master'], opts['retry_dns']) if salt.log.setup.is_console_configured(): log.error(msg) else: @@ -164,7 +164,7 @@ def resolve_dns(opts, fallback=True): time.sleep(opts['retry_dns']) try: ret['master_ip'] = salt.utils.dns_check( - opts['master'], True, opts['ipv6'] + opts['master'], opts['master_port'], True, opts['ipv6'], connect ) break except SaltClientError: @@ -689,7 +689,13 @@ class SMinion(MinionBase): def gen_modules(self, initial_load=False): ''' - Load all of the modules for the minion + Tell the minion to reload the execution modules + + CLI Example: + + .. code-block:: bash + + salt '*' sys.reload_modules ''' # Ensure that a pillar key is set in the opts, otherwise the loader # will pack a newly-generated empty dict as the __pillar__ dunder, and @@ -763,7 +769,13 @@ class MasterMinion(object): def gen_modules(self, initial_load=False): ''' - Load all of the modules for the minion + Tell the minion to reload the execution modules + + CLI Example: + + .. code-block:: bash + + salt '*' sys.reload_modules ''' self.utils = salt.loader.utils(self.opts) self.functions = salt.loader.minion_mods( @@ -864,22 +876,25 @@ class MinionManager(MinionBase): ''' last = 0 # never have we signed in auth_wait = minion.opts['acceptance_wait_time'] + failed = False while True: try: if minion.opts.get('beacons_before_connect', False): minion.setup_beacons() if minion.opts.get('scheduler_before_connect', False): minion.setup_scheduler() - yield minion.connect_master() + yield minion.connect_master(failed=failed) minion.tune_in(start=False) break except SaltClientError as exc: + failed = True log.error('Error while bringing up minion for multi-master. Is master at {0} responding?'.format(minion.opts['master'])) last = time.time() if auth_wait < self.max_auth_wait: auth_wait += self.auth_wait yield tornado.gen.sleep(auth_wait) # TODO: log? except Exception as e: + failed = True log.critical('Unexpected error while connecting to {0}'.format(minion.opts['master']), exc_info=True) # Multi Master Tune In @@ -1020,7 +1035,7 @@ class Minion(MinionBase): time.sleep(1) sys.exit(0) - def sync_connect_master(self, timeout=None): + def sync_connect_master(self, timeout=None, failed=False): ''' Block until we are connected to a master ''' @@ -1031,7 +1046,7 @@ class Minion(MinionBase): self._sync_connect_master_success = True self.io_loop.stop() - self._connect_master_future = self.connect_master() + self._connect_master_future = self.connect_master(failed=failed) # finish connecting to master self._connect_master_future.add_done_callback(on_connect_master_future_done) if timeout: @@ -1059,11 +1074,11 @@ class Minion(MinionBase): self.schedule.returners = self.returners @tornado.gen.coroutine - def connect_master(self): + def connect_master(self, failed=False): ''' Return a future which will complete when you are connected to a master ''' - master, self.pub_channel = yield self.eval_master(self.opts, self.timeout, self.safe) + master, self.pub_channel = yield self.eval_master(self.opts, self.timeout, self.safe, failed) yield self._post_master_init(master) # TODO: better name... @@ -2626,6 +2641,7 @@ class SyndicManager(MinionBase): ''' last = 0 # never have we signed in auth_wait = opts['acceptance_wait_time'] + failed = False while True: log.debug('Syndic attempting to connect to {0}'.format(opts['master'])) try: @@ -2634,7 +2650,7 @@ class SyndicManager(MinionBase): safe=False, io_loop=self.io_loop, ) - yield syndic.connect_master() + yield syndic.connect_master(failed=failed) # set up the syndic to handle publishes (specifically not event forwarding) syndic.tune_in_no_block() @@ -2644,6 +2660,7 @@ class SyndicManager(MinionBase): log.info('Syndic successfully connected to {0}'.format(opts['master'])) break except SaltClientError as exc: + failed = True log.error('Error while bringing up syndic for multi-syndic. Is master at {0} responding?'.format(opts['master'])) last = time.time() if auth_wait < self.max_auth_wait: @@ -2652,6 +2669,7 @@ class SyndicManager(MinionBase): except KeyboardInterrupt: raise except: # pylint: disable=W0702 + failed = True log.critical('Unexpected error while connecting to {0}'.format(opts['master']), exc_info=True) raise tornado.gen.Return(syndic) diff --git a/salt/modules/docker.py b/salt/modules/docker.py index b835abdcb3..5315120aac 100644 --- a/salt/modules/docker.py +++ b/salt/modules/docker.py @@ -183,6 +183,10 @@ Functions - :py:func:`docker.disconnect_container_from_network ` +- Salt Functions and States Execution + - :py:func:`docker.call ` + - :py:func:`docker.sls ` + - :py:func:`docker.sls_build ` .. _docker-execution-driver: @@ -571,6 +575,12 @@ VALID_CREATE_OPTS = { 'devices': { 'path': 'HostConfig:Devices', }, + 'ulimits': { + 'path': 'HostConfig:Ulimits', + 'min_docker': (1, 6, 0), + 'min_docker_py': (1, 2, 0), + 'default': [], + }, } @@ -831,10 +841,10 @@ def _get_client(timeout=None): 'Docker machine {0} failed: {1}'.format(docker_machine, exc)) try: - __context__['docker.client'] = docker.Client(**client_kwargs) - except AttributeError: # docker-py 2.0 renamed this client attribute __context__['docker.client'] = docker.APIClient(**client_kwargs) + except AttributeError: + __context__['docker.client'] = docker.Client(**client_kwargs) def _get_md5(name, path): @@ -1928,6 +1938,44 @@ def _validate_input(kwargs, else: kwargs['labels'] = salt.utils.repack_dictlist(kwargs['labels']) + def _valid_ulimits(): # pylint: disable=unused-variable + ''' + Must be a string or list of strings with bind mount information + ''' + if kwargs.get('ulimits') is None: + # No need to validate + return + err = ( + 'Invalid ulimits configuration. See the documentation for proper ' + 'usage.' + ) + try: + _valid_dictlist('ulimits') + # If this was successful then assume the correct API value was + # passed on on the CLI and do not proceed with validation. + return + except SaltInvocationError: + pass + try: + _valid_stringlist('ulimits') + except SaltInvocationError: + raise SaltInvocationError(err) + + new_ulimits = [] + for ulimit in kwargs['ulimits']: + ulimit_name, comps = ulimit.strip().split('=', 1) + try: + comps = [int(x) for x in comps.split(':', 1)] + except ValueError: + raise SaltInvocationError(err) + if len(comps) == 1: + comps *= 2 + soft_limit, hard_limit = comps + new_ulimits.append({'Name': ulimit_name, + 'Soft': soft_limit, + 'Hard': hard_limit}) + kwargs['ulimits'] = new_ulimits + # And now, the actual logic to perform the validation if 'docker.docker_version' not in __context__: # Have to call this func using the __salt__ dunder (instead of just @@ -5793,7 +5841,17 @@ def _gather_pillar(pillarenv, pillar_override, **grains): def call(name, function, *args, **kwargs): ''' - Executes a salt function inside a container + Executes a Salt function inside a running container + + .. versionadded:: 2016.11.0 + + The container does not need to have Salt installed, but Python is required. + + name + Container name or ID + + function + Salt execution module function CLI Example: @@ -5801,11 +5859,7 @@ def call(name, function, *args, **kwargs): salt myminion docker.call test.ping salt myminion test.arg arg1 arg2 key1=val1 - - The container does not need to have Salt installed, but Python - is required. - - .. versionadded:: 2016.11.0 + salt myminion dockerng.call compassionate_mirzakhani test.arg arg1 arg2 key1=val1 ''' # where to put the salt-thin @@ -5862,11 +5916,23 @@ def call(name, function, *args, **kwargs): def sls(name, mods=None, saltenv='base', **kwargs): ''' - Apply the highstate defined by the specified modules. + Apply the states defined by the specified SLS modules to the running + container - For example, if your master defines the states ``web`` and ``rails``, you - can apply them to a container: - states by doing: + .. versionadded:: 2016.11.0 + + The container does not need to have Salt installed, but Python is required. + + name + Container name or ID + + mods : None + A string containing comma-separated list of SLS with defined states to + apply to the container. + + saltenv : base + Specify the environment from which to retrieve the SLS indicated by the + `mods` parameter. CLI Example: @@ -5874,10 +5940,6 @@ def sls(name, mods=None, saltenv='base', **kwargs): salt myminion docker.sls compassionate_mirzakhani mods=rails,web - The container does not need to have Salt installed, but Python - is required. - - .. versionadded:: 2016.11.0 ''' mods = [item.strip() for item in mods.split(',')] if mods else [] @@ -5936,11 +5998,25 @@ def sls(name, mods=None, saltenv='base', **kwargs): def sls_build(name, base='opensuse/python', mods=None, saltenv='base', dryrun=False, **kwargs): ''' - Build a docker image using the specified sls modules and base image. + Build a Docker image using the specified SLS modules on top of base image - For example, if your master defines the states ``web`` and ``rails``, you - can build a docker image inside myminion that results of applying those - states by doing: + .. versionadded:: 2016.11.0 + + The base image does not need to have Salt installed, but Python is required. + + name + Image name to be built and committed + + base : opensuse/python + Name or ID of the base image + + mods : None + A string containing comma-separated list of SLS with defined states to + apply to the base image. + + saltenv : base + Specify the environment from which to retrieve the SLS indicated by the + `mods` parameter. base the base image @@ -5966,12 +6042,7 @@ def sls_build(name, base='opensuse/python', mods=None, saltenv='base', salt myminion docker.sls_build imgname base=mybase mods=rails,web - The base image does not need to have Salt installed, but Python - is required. - - .. versionadded:: 2016.11.0 ''' - create_kwargs = salt.utils.clean_kwargs(**copy.deepcopy(kwargs)) for key in ('image', 'name', 'cmd', 'interactive', 'tty'): try: @@ -6038,11 +6109,15 @@ def get_client_args(): except AttributeError: try: endpoint_config_args = \ - _argspec(docker.utils.create_endpoint_config).args + _argspec(docker.utils.utils.create_endpoint_config).args except AttributeError: - raise CommandExecutionError( - 'Failed to get create_host_config argspec' - ) + try: + endpoint_config_args = \ + _argspec(docker.utils.create_endpoint_config).args + except AttributeError: + raise CommandExecutionError( + 'Failed to get create_endpoint_config argspec' + ) for arglist in (config_args, host_config_args, endpoint_config_args): try: diff --git a/salt/modules/file.py b/salt/modules/file.py index 518d7bbbf4..e3bfedb97e 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -687,31 +687,54 @@ def check_hash(path, file_hash): ''' Check if a file matches the given hash string - Returns true if the hash matched, otherwise false. Raises ValueError if - the hash was not formatted correctly. + Returns ``True`` if the hash matches, otherwise ``False``. path - A file path + Path to a file local to the minion. + hash - A string in the form :. For example: - ``md5:e138491e9d5b97023cea823fe17bac22`` + The hash to check against the file specified in the ``path`` argument. + For versions 2016.11.4 and newer, the hash can be specified without an + accompanying hash type (e.g. ``e138491e9d5b97023cea823fe17bac22``), + but for earlier releases it is necessary to also specify the hash type + in the format ``:`` (e.g. + ``md5:e138491e9d5b97023cea823fe17bac22``). CLI Example: .. code-block:: bash - salt '*' file.check_hash /etc/fstab md5: + salt '*' file.check_hash /etc/fstab e138491e9d5b97023cea823fe17bac22 + salt '*' file.check_hash /etc/fstab md5:e138491e9d5b97023cea823fe17bac22 ''' path = os.path.expanduser(path) - hash_parts = file_hash.split(':', 1) - if len(hash_parts) != 2: - # Support "=" for backward compatibility. - hash_parts = file_hash.split('=', 1) - if len(hash_parts) != 2: - raise ValueError('Bad hash format: \'{0}\''.format(file_hash)) - hash_form, hash_value = hash_parts - return get_hash(path, hash_form) == hash_value + if not isinstance(file_hash, six.string_types): + raise SaltInvocationError('hash must be a string') + + for sep in (':', '='): + if sep in file_hash: + hash_type, hash_value = file_hash.split(sep, 1) + break + else: + hash_value = file_hash + hash_len = len(file_hash) + hash_type = HASHES_REVMAP.get(hash_len) + if hash_type is None: + raise SaltInvocationError( + 'Hash {0} (length: {1}) could not be matched to a supported ' + 'hash type. The supported hash types and lengths are: ' + '{2}'.format( + file_hash, + hash_len, + ', '.join( + ['{0} ({1})'.format(HASHES_REVMAP[x], x) + for x in sorted(HASHES_REVMAP)] + ), + ) + ) + + return get_hash(path, hash_type) == hash_value def find(path, *args, **kwargs): diff --git a/salt/modules/openscap.py b/salt/modules/openscap.py index b50ede7c28..2061550012 100644 --- a/salt/modules/openscap.py +++ b/salt/modules/openscap.py @@ -78,6 +78,7 @@ def xccdf(params): error = None upload_dir = None action = None + returncode = None try: parser = _ArgumentParser() @@ -92,14 +93,17 @@ def xccdf(params): tempdir = tempfile.mkdtemp() proc = Popen( shlex.split(cmd), stdout=PIPE, stderr=PIPE, cwd=tempdir) - (stdoutdata, stderrdata) = proc.communicate() + (stdoutdata, error) = proc.communicate() success = _OSCAP_EXIT_CODES_MAP[proc.returncode] + returncode = proc.returncode if success: caller = Caller() caller.cmd('cp.push_dir', tempdir) shutil.rmtree(tempdir, ignore_errors=True) upload_dir = tempdir - else: - error = stderrdata - return dict(success=success, upload_dir=upload_dir, error=error) + return dict( + success=success, + upload_dir=upload_dir, + error=error, + returncode=returncode) diff --git a/salt/modules/service.py b/salt/modules/service.py index bb7133ee99..49186e4c9d 100644 --- a/salt/modules/service.py +++ b/salt/modules/service.py @@ -53,7 +53,7 @@ def __virtual__(): if __grains__['kernel'] != 'Linux': return (False, 'Non Linux OSes are not supported') # SUSE >=12.0 uses systemd - if __grains__.get('os_family', '') == 'SUSE': + if __grains__.get('os_family', '') == 'Suse': try: # osrelease might be in decimal format (e.g. "12.1"), or for # SLES might include service pack (e.g. "11 SP3"), so split on diff --git a/salt/modules/sysmod.py b/salt/modules/sysmod.py index 574c8d1aa2..a7c731e93e 100644 --- a/salt/modules/sysmod.py +++ b/salt/modules/sysmod.py @@ -421,8 +421,10 @@ def reload_modules(): salt '*' sys.reload_modules ''' - # This is handled inside the minion.py file, the function is caught before - # it ever gets here + # This function is actually handled inside the minion.py file, the function + # is caught before it ever gets here. Therefore, the docstring above is + # only for the online docs, and ANY CHANGES made to it must also be made in + # each of the gen_modules() funcs in minion.py. return True diff --git a/salt/modules/win_iis.py b/salt/modules/win_iis.py index 9cc7792d34..3e873076e0 100644 --- a/salt/modules/win_iis.py +++ b/salt/modules/win_iis.py @@ -15,6 +15,7 @@ from __future__ import absolute_import, unicode_literals import json import logging import os +import decimal # Import salt libs from salt.ext.six.moves import range @@ -108,6 +109,22 @@ def _list_certs(certificate_store='My'): return ret +def _iisVersion(): + pscmd = [] + pscmd.append(r"Get-ItemProperty HKLM:\\SOFTWARE\\Microsoft\\InetStp\\") + pscmd.append(' | Select-Object MajorVersion, MinorVersion') + + cmd_ret = _srvmgr(pscmd, return_json=True) + + try: + items = json.loads(cmd_ret['stdout'], strict=False) + except ValueError: + log.error('Unable to parse return data as Json.') + return -1 + + return decimal.Decimal("{0}.{1}".format(items[0]['MajorVersion'], items[0]['MinorVersion'])) + + def _srvmgr(cmd, return_json=False): ''' Execute a powershell command from the WebAdministration PS module. @@ -765,6 +782,11 @@ def create_cert_binding(name, site, hostheader='', ipaddress='*', port=443, ''' name = str(name).upper() binding_info = _get_binding_info(hostheader, ipaddress, port) + + if _iisVersion() < 8: + # IIS 7.5 and earlier don't support SNI for HTTPS, therefore cert bindings don't contain the host header + binding_info = binding_info.rpartition(':')[0] + ':' + binding_path = r"IIS:\SslBindings\{0}".format(binding_info.replace(':', '!')) if sslflags not in _VALID_SSL_FLAGS: @@ -801,10 +823,19 @@ def create_cert_binding(name, site, hostheader='', ipaddress='*', port=443, log.error('Certificate not present: {0}'.format(name)) return False - ps_cmd = ['New-Item', - '-Path', "'{0}'".format(binding_path), - '-Thumbprint', "'{0}'".format(name), - '-SSLFlags', '{0}'.format(sslflags)] + if _iisVersion() < 8: + # IIS 7.5 and earlier have different syntax for associating a certificate with a site + # Modify IP spec to IIS 7.5 format + iis7path = binding_path.replace(r"\*!", "\\0.0.0.0!") + + ps_cmd = ['New-Item', + '-Path', "'{0}'".format(iis7path), + '-Thumbprint', "'{0}'".format(name)] + else: + ps_cmd = ['New-Item', + '-Path', "'{0}'".format(binding_path), + '-Thumbprint', "'{0}'".format(name), + '-SSLFlags', '{0}'.format(sslflags)] cmd_ret = _srvmgr(ps_cmd) @@ -824,6 +855,7 @@ def create_cert_binding(name, site, hostheader='', ipaddress='*', port=443, return True log.error('Unable to create certificate binding: {0}'.format(name)) + return False diff --git a/salt/output/nested.py b/salt/output/nested.py index 884ad693b9..7d21dba49f 100644 --- a/salt/output/nested.py +++ b/salt/output/nested.py @@ -25,7 +25,7 @@ Example output:: ''' from __future__ import absolute_import # Import python libs -import collections +import salt.utils.odict from numbers import Number # Import salt libs @@ -130,7 +130,7 @@ class NestDisplay(object): ) # respect key ordering of ordered dicts - if isinstance(ret, collections.OrderedDict): + if isinstance(ret, salt.utils.odict.OrderedDict): keys = ret.keys() else: keys = sorted(ret) diff --git a/salt/pillar/__init__.py b/salt/pillar/__init__.py index bf966239d7..56ced2069b 100644 --- a/salt/pillar/__init__.py +++ b/salt/pillar/__init__.py @@ -405,20 +405,21 @@ class Pillar(object): # Gather initial top files try: if self.opts['pillarenv']: - tops[self.opts['pillarenv']] = [ - compile_template( - self.client.cache_file( - self.opts['state_top'], - self.opts['pillarenv'] - ), - self.rend, - self.opts['renderer'], - self.opts['renderer_blacklist'], - self.opts['renderer_whitelist'], - self.opts['pillarenv'], - _pillar_rend=True, - ) - ] + top = self.client.cache_file( + self.opts['state_top'], self.opts['pillarenv']) + + if top: + tops[self.opts['pillarenv']] = [ + compile_template( + top, + self.rend, + self.opts['renderer'], + self.opts['renderer_blacklist'], + self.opts['renderer_whitelist'], + self.opts['pillarenv'], + _pillar_rend=True, + ) + ] else: for saltenv in self._get_envs(): if self.opts.get('pillar_source_merging_strategy', None) == "none": diff --git a/salt/returners/local_cache.py b/salt/returners/local_cache.py index d46cb1e70c..c2b58c1a14 100644 --- a/salt/returners/local_cache.py +++ b/salt/returners/local_cache.py @@ -239,6 +239,9 @@ def save_minions(jid, minions, syndic_id=None): ''' Save/update the serialized list of minions for a given job ''' + # Ensure we have a list for Python 3 compatability + minions = list(minions) + log.debug( 'Adding minions for job %s%s: %s', jid, diff --git a/salt/states/docker.py b/salt/states/docker.py index 8e0c5de94a..9bb376327c 100644 --- a/salt/states/docker.py +++ b/salt/states/docker.py @@ -32,7 +32,7 @@ from salt.modules.docker import ( STOP_TIMEOUT, VALID_CREATE_OPTS, _validate_input, - _get_repo_tag + _get_repo_tag, ) # pylint: enable=no-name-in-module,import-error import salt.utils @@ -223,6 +223,7 @@ def _compare(actual, create_kwargs, defaults_from_image): ret.update({item: {'old': actual_ports, 'new': desired_ports}}) continue + elif item == 'volumes': if actual_data is None: actual_data = [] @@ -390,6 +391,7 @@ def _compare(actual, create_kwargs, defaults_from_image): # sometimes `[]`. We have to deal with it. if bool(actual_data) != bool(data): ret.update({item: {'old': actual_data, 'new': data}}) + elif item == 'labels': if actual_data is None: actual_data = {} @@ -416,6 +418,7 @@ def _compare(actual, create_kwargs, defaults_from_image): if data != actual_data: ret.update({item: {'old': actual_data, 'new': data}}) continue + elif item == 'devices': if data: keys = ['PathOnHost', 'PathInContainer', 'CgroupPermissions'] @@ -433,6 +436,7 @@ def _compare(actual, create_kwargs, defaults_from_image): if data != actual_data: ret.update({item: {'old': actual_data, 'new': data}}) continue + elif item == 'security_opt': if actual_data is None: actual_data = [] @@ -448,6 +452,7 @@ def _compare(actual, create_kwargs, defaults_from_image): ret.update({item: {'old': actual_data, 'new': desired_data}}) continue + elif item in ('cmd', 'command', 'entrypoint'): if (actual_data is None and item not in create_kwargs and _image_get(config['image_path'])): @@ -1567,6 +1572,29 @@ def running(name, This option requires Docker 1.5.0 or newer. + ulimits + List of ulimits. These limits should be passed in + the format ``::``, with the hard + limit being optional. + + .. code-block:: yaml + + foo: + dockerng.running: + - image: bar/baz:latest + - ulimits: nofile=1024:1024,nproc=60 + + Ulimits can be passed as a YAML list instead of a comma-separated list: + + .. code-block:: yaml + + foo: + dockerng.running: + - image: bar/baz:latest + - ulimits: + - nofile=1024:1024 + - nproc=60 + labels Add Metadata to the container. Can be a list of strings/dictionaries or a dictionary of strings (keys and values). diff --git a/salt/states/file.py b/salt/states/file.py index 77a68140cb..f8c75621dc 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -283,7 +283,8 @@ import salt.utils.files import salt.utils.templates import salt.utils.url from salt.utils.locales import sdecode -from salt.exceptions import CommandExecutionError +from salt.exceptions import CommandExecutionError, SaltInvocationError + if salt.utils.is_windows(): import salt.utils.win_dacl @@ -1446,7 +1447,7 @@ def managed(name, will not be changed or managed. If the file is hosted on a HTTP or FTP server then the source_hash - argument is also required + argument is also required. A list of sources can also be passed in to provide a default source and a set of fallbacks. The first source in the list that is found to exist @@ -3542,8 +3543,7 @@ def line(name, content=None, match=None, mode=None, location=None, if not name: return _error(ret, 'Must provide name to file.line') - if create and not os.path.isfile(name): - managed(name, create=create, user=user, group=group, mode=file_mode) + managed(name, create=create, user=user, group=group, mode=file_mode) check_res, check_msg = _check_file(name) if not check_res: @@ -3822,7 +3822,7 @@ def blockreplace( will not be changed or managed. If the file is hosted on a HTTP or FTP server then the source_hash - argument is also required + argument is also required. A list of sources can also be passed in to provide a default source and a set of fallbacks. The first source in the list that is found to exist @@ -3831,7 +3831,8 @@ def blockreplace( .. code-block:: yaml file_override_example: - file.managed: + file.blockreplace: + - name: /etc/example.conf - source: - salt://file_that_does_not_exist - salt://file_that_exists @@ -3856,45 +3857,8 @@ def blockreplace( sha1 40 md5 32 - **Using a Source Hash File** - The file can contain several checksums for several files. Each line - must contain both the file name and the hash. If no file name is - matched, the first hash encountered will be used, otherwise the most - secure hash with the correct source file name will be used. - - When using a source hash file the source_hash argument needs to be a - url, the standard download urls are supported, ftp, http, salt etc: - - Example: - - .. code-block:: yaml - - tomdroid-src-0.7.3.tar.gz: - file.managed: - - name: /tmp/tomdroid-src-0.7.3.tar.gz - - source: https://launchpad.net/tomdroid/beta/0.7.3/+download/tomdroid-src-0.7.3.tar.gz - - source_hash: https://launchpad.net/tomdroid/beta/0.7.3/+download/tomdroid-src-0.7.3.hash - - The following is an example of the supported source_hash format: - - .. code-block:: text - - /etc/rc.conf ef6e82e4006dee563d98ada2a2a80a27 - sha254c8525aee419eb649f0233be91c151178b30f0dff8ebbdcc8de71b1d5c8bcc06a /etc/resolv.conf - ead48423703509d37c4a90e6a0d53e143b6fc268 - - Debian file type ``*.dsc`` files are also supported. - - **Inserting the Source Hash in the sls Data** - Examples: - - .. code-block:: yaml - - tomdroid-src-0.7.3.tar.gz: - file.managed: - - name: /tmp/tomdroid-src-0.7.3.tar.gz - - source: https://launchpad.net/tomdroid/beta/0.7.3/+download/tomdroid-src-0.7.3.tar.gz - - source_hash: md5=79eef25f9b0b2c642c62b7f737d4f53f + See the ``source_hash`` parameter description for :mod:`file.managed + ` function for more details and examples. template The named templating engine will be used to render the downloaded file. @@ -4349,34 +4313,8 @@ def append(name, sha1 40 md5 32 - The file can contain several checksums for several files. Each line - must contain both the file name and the hash. If no file name is - matched, the first hash encountered will be used, otherwise the most - secure hash with the correct source file name will be used. - - Debian file type ``*.dsc`` is supported. - - Examples: - - .. code-block:: text - - /etc/rc.conf ef6e82e4006dee563d98ada2a2a80a27 - sha254c8525aee419eb649f0233be91c151178b30f0dff8ebbdcc8de71b1d5c8bcc06a /etc/resolv.conf - ead48423703509d37c4a90e6a0d53e143b6fc268 - - Known issues: - If the remote server URL has the hash file as an apparent - sub-directory of the source file, the module will discover that it - has already cached a directory where a file should be cached. For - example: - - .. code-block:: yaml - - tomdroid-src-0.7.3.tar.gz: - file.managed: - - name: /tmp/tomdroid-src-0.7.3.tar.gz - - source: https://launchpad.net/tomdroid/beta/0.7.3/+download/tomdroid-src-0.7.3.tar.gz - - source_hash: https://launchpad.net/tomdroid/beta/0.7.3/+download/tomdroid-src-0.7.3.tar.gz/+md5 + See the ``source_hash`` parameter description for :mod:`file.managed + ` function for more details and examples. template The named templating engine will be used to render the appended-to file. @@ -4790,7 +4728,6 @@ def prepend(name, def patch(name, source=None, - hash=None, options='', dry_run_first=True, **kwargs): @@ -4812,11 +4749,13 @@ def patch(name, salt://spam/eggs. A source is required. hash - Hash of the patched file. If the hash of the target file matches this - value then the patch is assumed to have been applied. The hash string - is as string in the form :. For example: - md5:e138491e9d5b97023cea823fe17bac22. For more informations, check the - :mod:`file.check_hash ` module. + The hash of the patched file. If the hash of the target file matches + this value then the patch is assumed to have been applied. For versions + 2016.11.4 and newer, the hash can be specified without an accompanying + hash type (e.g. ``e138491e9d5b97023cea823fe17bac22``), but for earlier + releases it is necessary to also specify the hash type in the format + ``:`` (e.g. + ``md5:e138491e9d5b97023cea823fe17bac22``). options Extra options to pass to patch. @@ -4829,7 +4768,7 @@ def patch(name, by the ``source`` parameter. If not provided, this defaults to the environment from which the state is being executed. - Usage: + **Usage:** .. code-block:: yaml @@ -4837,8 +4776,15 @@ def patch(name, /opt/file.txt: file.patch: - source: salt://file.patch - - hash: md5:e138491e9d5b97023cea823fe17bac22 + - hash: e138491e9d5b97023cea823fe17bac22 + + .. note:: + For minions running version 2016.11.3 or older, the hash in the example + above would need to be specified with the hash type (i.e. + ``md5:e138491e9d5b97023cea823fe17bac22``). ''' + hash_ = kwargs.pop('hash', None) + if 'env' in kwargs: salt.utils.warn_until( 'Oxygen', @@ -4858,11 +4804,16 @@ def patch(name, return _error(ret, check_msg) if not source: return _error(ret, 'Source is required') - if hash is None: + if hash_ is None: return _error(ret, 'Hash is required') - if hash and __salt__['file.check_hash'](name, hash): - ret.update(result=True, comment='Patch is already applied') + try: + if hash_ and __salt__['file.check_hash'](name, hash_): + ret['result'] = True + ret['comment'] = 'Patch is already applied' + return ret + except (SaltInvocationError, ValueError) as exc: + ret['comment'] = exc.__str__() return ret # get cached file or copy it to cache @@ -4873,9 +4824,8 @@ def patch(name, return ret log.debug( - 'State patch.applied cached source {0} -> {1}'.format( - source, cached_source_path - ) + 'State patch.applied cached source %s -> %s', + source, cached_source_path ) if dry_run_first or __opts__['test']: @@ -4886,20 +4836,18 @@ def patch(name, ret['comment'] = 'File {0} will be patched'.format(name) ret['result'] = None return ret - if ret['changes']['retcode']: + if ret['changes']['retcode'] != 0: return ret ret['changes'] = __salt__['file.patch']( name, cached_source_path, options=options ) - ret['result'] = not ret['changes']['retcode'] - if ret['result'] and hash and not __salt__['file.check_hash'](name, hash): - ret.update( - result=False, - comment='File {0} hash mismatch after patch was applied'.format( - name - ) - ) + ret['result'] = ret['changes']['retcode'] == 0 + # No need to check for SaltInvocationError or ValueError this time, since + # these exceptions would have been caught above. + if ret['result'] and hash_ and not __salt__['file.check_hash'](name, hash_): + ret['result'] = False + ret['comment'] = 'Hash mismatch after patch was applied' return ret diff --git a/salt/states/ssh_known_hosts.py b/salt/states/ssh_known_hosts.py index 93cb786f6c..0de1414912 100644 --- a/salt/states/ssh_known_hosts.py +++ b/salt/states/ssh_known_hosts.py @@ -25,6 +25,20 @@ import os # Import salt libs from salt.exceptions import CommandNotFoundError +import salt.utils + +# Define the state's virtual name +__virtualname__ = 'ssh_known_hosts' + + +def __virtual__(): + ''' + Does not work on Windows, requires ssh module functions + ''' + if salt.utils.is_windows(): + return False, 'ssh_known_hosts: Does not support Windows' + + return __virtualname__ def present( diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py index f7d78906ba..2311ea8f02 100644 --- a/salt/transport/zeromq.py +++ b/salt/transport/zeromq.py @@ -351,7 +351,7 @@ class AsyncZeroMQPubChannel(salt.transport.mixins.auth.AESPubClientMixin, salt.t zmq.RECONNECT_IVL_MAX, self.opts['recon_max'] ) - if self.opts['ipv6'] is True and hasattr(zmq, 'IPV4ONLY'): + if (self.opts['ipv6'] is True or ':' in self.opts['master_ip']) and hasattr(zmq, 'IPV4ONLY'): # IPv6 sockets work for both IPv6 and IPv4 addresses self._socket.setsockopt(zmq.IPV4ONLY, 0) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 3e77f8bf72..ead6252c30 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -751,10 +751,11 @@ def ip_bracket(addr): return addr -def dns_check(addr, safe=False, ipv6=False): +def dns_check(addr, port, safe=False, ipv6=None, connect=True): ''' Return the ip resolved by dns, but do not exit on failure, only raise an exception. Obeys system preference for IPv4/6 address resolution. + Tries to connect to the address before considering it useful. ''' error = False lookup = addr @@ -769,18 +770,30 @@ def dns_check(addr, safe=False, ipv6=False): if not hostnames: error = True else: - addr = False + resolved = False for h in hostnames: - if h[0] == socket.AF_INET: - addr = ip_bracket(h[4][0]) + if h[0] == socket.AF_INET and ipv6 is True: + continue + if h[0] == socket.AF_INET6 and ipv6 is False: + continue + if h[0] == socket.AF_INET6 and connect is False and ipv6 is None: + continue + + candidate_addr = ip_bracket(h[4][0]) + + if not connect: + resolved = candidate_addr + + s = socket.socket(h[0], socket.SOCK_STREAM) + try: + s.connect((candidate_addr.strip('[]'), port)) + s.close() + + resolved = candidate_addr break - elif h[0] == socket.AF_INET6: - if not ipv6: - seen_ipv6 = True - continue - addr = ip_bracket(h[4][0]) - break - if not addr: + except socket.error: + pass + if not resolved: error = True except TypeError: err = ('Attempt to resolve address \'{0}\' failed. Invalid or unresolveable address').format(lookup) @@ -789,10 +802,7 @@ def dns_check(addr, safe=False, ipv6=False): error = True if error: - if seen_ipv6 and not addr: - err = ('DNS lookup of \'{0}\' failed, but ipv6 address ignored. Enable ipv6 in config to use it.').format(lookup) - else: - err = ('DNS lookup of \'{0}\' failed.').format(lookup) + err = ('DNS lookup or connection check of \'{0}\' failed.').format(addr) if safe: if salt.log.is_console_configured(): # If logging is not configured it also means that either @@ -801,7 +811,7 @@ def dns_check(addr, safe=False, ipv6=False): log.error(err) raise SaltClientError() raise SaltSystemExit(code=42, msg=err) - return addr + return resolved def required_module_list(docstring=None): diff --git a/salt/utils/context.py b/salt/utils/context.py index b1c8c381bd..62934cd07a 100644 --- a/salt/utils/context.py +++ b/salt/utils/context.py @@ -66,12 +66,15 @@ class ContextDict(collections.MutableMapping): then allow any children to override the values of the parent. ''' - def __init__(self, **data): + def __init__(self, threadsafe=False, **data): # state should be thread local, so this object can be threadsafe self._state = threading.local() # variable for the overridden data self._state.data = None self.global_data = {} + # Threadsafety indicates whether or not we should protect data stored + # in child context dicts from being leaked + self._threadsafe = threadsafe @property def active(self): @@ -89,7 +92,7 @@ class ContextDict(collections.MutableMapping): ''' Clone this context, and return the ChildContextDict ''' - child = ChildContextDict(parent=self, overrides=kwargs) + child = ChildContextDict(parent=self, threadsafe=self._threadsafe, overrides=kwargs) return child def __setitem__(self, key, val): @@ -127,19 +130,24 @@ class ChildContextDict(collections.MutableMapping): '''An overrideable child of ContextDict ''' - def __init__(self, parent, overrides=None): + def __init__(self, parent, overrides=None, threadsafe=False): self.parent = parent self._data = {} if overrides is None else overrides self._old_data = None # merge self.global_data into self._data - for k, v in six.iteritems(self.parent.global_data): - if k not in self._data: - # A deepcopy is necessary to avoid using the same - # objects in globals as we do in thread local storage. - # Otherwise, changing one would automatically affect - # the other. - self._data[k] = copy.deepcopy(v) + if threadsafe: + for k, v in six.iteritems(self.parent.global_data): + if k not in self._data: + # A deepcopy is necessary to avoid using the same + # objects in globals as we do in thread local storage. + # Otherwise, changing one would automatically affect + # the other. + self._data[k] = copy.deepcopy(v) + else: + for k, v in six.iteritems(self.parent.global_data): + if k not in self._data: + self._data[k] = v def __setitem__(self, key, val): self._data[key] = val diff --git a/tests/integration/files/file/base/_modules/runtests_helpers.py b/tests/integration/files/file/base/_modules/runtests_helpers.py index 238b79cc41..5968aab7c7 100644 --- a/tests/integration/files/file/base/_modules/runtests_helpers.py +++ b/tests/integration/files/file/base/_modules/runtests_helpers.py @@ -9,12 +9,16 @@ # Import python libs from __future__ import absolute_import +import fnmatch import os +import re import tempfile # Import salt libs import salt.utils +# Import 3rd-party libs +import salt.ext.six as six SYS_TMP_DIR = os.path.realpath( # Avoid ${TMPDIR} and gettempdir() on MacOS as they yield a base path too long @@ -29,9 +33,75 @@ TMP = os.path.join(SYS_TMP_DIR, 'salt-tests-tmpdir') def get_salt_temp_dir(): return TMP + def get_salt_temp_dir_for_path(*path): return os.path.join(TMP, *path) def get_sys_temp_dir_for_path(*path): return os.path.join(SYS_TMP_DIR, *path) + + +def get_invalid_docs(): + ''' + Outputs the functions which do not have valid CLI example, or are missing a + docstring. + ''' + allow_failure = ( + 'cmd.win_runas', + 'cp.recv', + 'glance.warn_until', + 'ipset.long_range', + 'libcloud_dns.get_driver', + 'log.critical', + 'log.debug', + 'log.error', + 'log.exception', + 'log.info', + 'log.warning', + 'lowpkg.bin_pkg_info', + 'lxc.run_cmd', + 'nspawn.restart', + 'nspawn.stop', + 'pkg.expand_repo_def', + 'pip.iteritems', + 'runtests_decorators.depends', + 'runtests_decorators.depends_will_fallback', + 'runtests_decorators.missing_depends', + 'runtests_decorators.missing_depends_will_fallback', + 'state.apply', + 'status.list2cmdline', + 'swift.head', + 'travisci.parse_qs', + 'vsphere.clean_kwargs', + 'vsphere.disconnect', + 'vsphere.get_service_instance_via_proxy', + 'vsphere.gets_service_instance_via_proxy', + 'vsphere.supports_proxies', + 'vsphere.test_vcenter_connection', + 'vsphere.wraps', + ) + allow_failure_glob = ( + 'runtests_helpers.*', + ) + nodoc = set() + noexample = set() + for fun, docstring in six.iteritems(__salt__['sys.doc']()): + if fun in allow_failure: + continue + else: + for pat in allow_failure_glob: + if fnmatch.fnmatch(fun, pat): + matched_glob = True + break + else: + matched_glob = False + if matched_glob: + continue + if not isinstance(docstring, six.string_types): + nodoc.add(fun) + elif isinstance(docstring, dict) and not re.search(r'([E|e]xample(?:s)?)+(?:.*):?', docstring): + noexample.add(fun) + + return {'missing_docstring': sorted(nodoc), + 'missing_cli_example': sorted(noexample)} diff --git a/tests/integration/modules/test_sysmod.py b/tests/integration/modules/test_sysmod.py index a2bfb8ff44..5f351a8491 100644 --- a/tests/integration/modules/test_sysmod.py +++ b/tests/integration/modules/test_sysmod.py @@ -2,10 +2,6 @@ # Import python libs from __future__ import absolute_import -import logging -import re - -log = logging.getLogger(__name__) # Import Salt Testing libs from salttesting.helpers import ensure_in_syspath @@ -14,9 +10,6 @@ ensure_in_syspath('../../') # Import salt libs import integration -# Import 3rd-party libs -import salt.ext.six as six - class SysModuleTest(integration.ModuleCase): ''' @@ -26,93 +19,15 @@ class SysModuleTest(integration.ModuleCase): ''' Make sure no functions are exposed that don't have valid docstrings ''' - mods = self.run_function('sys.list_modules') - nodoc = set() - noexample = set() - allow_failure = ( - 'container_resource.run', - 'cmd.win_runas', - 'cp.recv', - 'glance.warn_until', - 'ipset.long_range', - 'libcloud_dns.get_driver', - 'log.critical', - 'log.debug', - 'log.error', - 'log.exception', - 'log.info', - 'log.warning', - 'lowpkg.bin_pkg_info', - 'lxc.run_cmd', - 'nspawn.restart', - 'nspawn.stop', - 'pkg.expand_repo_def', - 'pip.iteritems', - 'runtests_decorators.depends', - 'runtests_decorators.depends_will_fallback', - 'runtests_decorators.missing_depends', - 'runtests_decorators.missing_depends_will_fallback', - 'state.apply', - 'status.list2cmdline', - 'swift.head', - 'travisci.parse_qs', - 'vsphere.clean_kwargs', - 'vsphere.disconnect', - 'vsphere.get_service_instance_via_proxy', - 'vsphere.gets_service_instance_via_proxy', - 'vsphere.supports_proxies', - 'vsphere.test_vcenter_connection', - 'vsphere.wraps', - 'yumpkg.expand_repo_def', - 'yumpkg5.expand_repo_def', - ) - - batches = 2 - mod_count = len(mods) - batch_size = mod_count / float(batches) - if batch_size.is_integer(): - batch_size = int(batch_size) - else: - # Check if the module count is evenly divisible by the number of - # batches. If not, increase the batch_size by the number of batches - # being run. This ensures that we get the correct number of - # batches, and that we don't end up running sys.doc an extra time - # to cover the remainder. For example, if we had a batch count of 2 - # and 121 modules, if we just divided by 2 we'd end up running - # sys.doc 3 times. - batch_size = int(batch_size) + batches - - log.debug('test_valid_docs batch size = %s', batch_size) - start = 0 - end = batch_size - while start <= mod_count: - log.debug('running sys.doc on mods[%s:%s]', start, end) - docs = self.run_function('sys.doc', mods[start:end]) - if docs == 'VALUE TRIMMED': - self.fail( - 'sys.doc output trimmed. It may be necessary to increase ' - 'the number of batches' - ) - for fun in docs: - if fun.startswith('runtests_helpers'): - continue - if fun in allow_failure: - continue - if isinstance(docs, dict) and not isinstance(docs[fun], six.string_types): - nodoc.add(fun) - elif isinstance(docs, dict) and not re.search(r'([E|e]xample(?:s)?)+(?:.*)::?', docs[fun]): - noexample.add(fun) - start += batch_size - end += batch_size - - if not nodoc and not noexample: + ret = self.run_function('runtests_helpers.get_invalid_docs') + if ret == {'missing_docstring': [], 'missing_cli_example': []}: return raise AssertionError( 'There are some functions which do not have a docstring or do not ' 'have an example:\nNo docstring:\n{0}\nNo example:\n{1}\n'.format( - '\n'.join([' - {0}'.format(f) for f in sorted(nodoc)]), - '\n'.join([' - {0}'.format(f) for f in sorted(noexample)]), + '\n'.join([' - {0}'.format(f) for f in ret['missing_docstring']]), + '\n'.join([' - {0}'.format(f) for f in ret['missing_cli_example']]), ) ) diff --git a/tests/integration/states/test_file.py b/tests/integration/states/test_file.py index 1f5efe66c1..a70c50037c 100644 --- a/tests/integration/states/test_file.py +++ b/tests/integration/states/test_file.py @@ -1819,7 +1819,7 @@ class FileTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn): src_file, ret = self.do_patch('hello_dolly') self.assertSaltFalseReturn(ret) self.assertInSaltComment( - 'File {0} hash mismatch after patch was applied'.format(src_file), + 'Hash mismatch after patch was applied', ret ) diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index b943c8a24d..d21e4496f2 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -468,7 +468,7 @@ class ConfigTestCase(TestCase, integration.AdaptedConfigurationTestCaseMixIn): syndic_opts = sconfig.syndic_config( syndic_conf_path, minion_conf_path ) - syndic_opts.update(salt.minion.resolve_dns(syndic_opts)) + syndic_opts.update(salt.minion.resolve_dns(syndic_opts, connect=False)) root_dir = syndic_opts['root_dir'] # id & pki dir are shared & so configured on the minion side self.assertEqual(syndic_opts['id'], 'minion') diff --git a/tests/unit/modules/test_openscap.py b/tests/unit/modules/test_openscap.py index c883975690..327af75238 100644 --- a/tests/unit/modules/test_openscap.py +++ b/tests/unit/modules/test_openscap.py @@ -72,7 +72,9 @@ class OpenscapTestCase(TestCase): response, { 'upload_dir': self.random_temp_dir, - 'error': None, 'success': True + 'error': '', + 'success': True, + 'returncode': 0 } ) @@ -80,7 +82,7 @@ class OpenscapTestCase(TestCase): 'salt.modules.openscap.Popen', MagicMock( return_value=Mock( - **{'returncode': 2, 'communicate.return_value': ('', '')} + **{'returncode': 2, 'communicate.return_value': ('', 'some error')} ) ) ) @@ -111,8 +113,9 @@ class OpenscapTestCase(TestCase): response, { 'upload_dir': self.random_temp_dir, - 'error': None, - 'success': True + 'error': 'some error', + 'success': True, + 'returncode': 2 } ) @@ -124,7 +127,8 @@ class OpenscapTestCase(TestCase): { 'error': 'argument --profile is required', 'upload_dir': None, - 'success': False + 'success': False, + 'returncode': None } ) @@ -132,7 +136,7 @@ class OpenscapTestCase(TestCase): 'salt.modules.openscap.Popen', MagicMock( return_value=Mock( - **{'returncode': 2, 'communicate.return_value': ('', '')} + **{'returncode': 2, 'communicate.return_value': ('', 'some error')} ) ) ) @@ -143,8 +147,9 @@ class OpenscapTestCase(TestCase): response, { 'upload_dir': self.random_temp_dir, - 'error': None, - 'success': True + 'error': 'some error', + 'success': True, + 'returncode': 2 } ) expected_cmd = [ @@ -181,19 +186,11 @@ class OpenscapTestCase(TestCase): { 'upload_dir': None, 'error': 'evaluation error', - 'success': False + 'success': False, + 'returncode': 1 } ) - @patch( - 'salt.modules.openscap.Popen', - MagicMock( - return_value=Mock(**{ - 'returncode': 1, - 'communicate.return_value': ('', 'evaluation error') - }) - ) - ) def test_openscap_xccdf_eval_fail_not_implemented_action(self): response = openscap.xccdf('info {0}'.format(self.policy_file)) @@ -202,6 +199,7 @@ class OpenscapTestCase(TestCase): { 'upload_dir': None, 'error': "argument action: invalid choice: 'info' (choose from 'eval')", - 'success': False + 'success': False, + 'returncode': None } ) diff --git a/tests/unit/modules/test_win_iis.py b/tests/unit/modules/test_win_iis.py index 98b81f3a88..a3347cab24 100644 --- a/tests/unit/modules/test_win_iis.py +++ b/tests/unit/modules/test_win_iis.py @@ -344,7 +344,8 @@ class WinIisTestCase(TestCase): @patch('salt.modules.win_iis._list_certs', MagicMock(return_value={'9988776655443322111000AAABBBCCCDDDEEEFFF': None})) @patch('salt.modules.win_iis._srvmgr', - MagicMock(return_value={'retcode': 0})) + MagicMock(return_value={'retcode': 0, 'stdout': 10})) + @patch('json.loads', MagicMock(return_value=[{'MajorVersion': 10, 'MinorVersion': 0}])) @patch('salt.modules.win_iis.list_bindings', MagicMock(return_value=BINDING_LIST)) @patch('salt.modules.win_iis.list_cert_bindings',