diff --git a/doc/faq.rst b/doc/faq.rst index b849540f29..1e2300cb31 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -14,10 +14,10 @@ open and proprietary projects. To expand on this a little: -There is much argument over the actual definition of "open core". From our standpoint, Salt is open source because +There is much argument over the actual definition of "open core". From our standpoint, Salt is open source because 1. It is a standalone product that anyone is free to use. -2. It is developed in the open with contributions accepted from the community for the good of the project. +2. It is developed in the open with contributions accepted from the community for the good of the project. 3. There are no features of Salt itself that are restricted to separate proprietary products distributed by SaltStack, Inc. 4. Because of our Apache 2.0 license, Salt can be used as the foundation for a project or even a proprietary tool. 5. Our APIs are open and documented (any lack of documentation is an oversight as opposed to an intentional decision by SaltStack the company) and available for use by anyone. diff --git a/requirements/windows.txt b/requirements/windows.txt new file mode 100644 index 0000000000..ddbe31dc27 --- /dev/null +++ b/requirements/windows.txt @@ -0,0 +1,10 @@ +backports-abc +backports.ssl-match-hostname +certifi +psutil +python-dateutil +pypiwin32 +pyzmq +six +timelib +WMI diff --git a/salt/client/__init__.py b/salt/client/__init__.py index 6d63d43338..eb93591d5a 100644 --- a/salt/client/__init__.py +++ b/salt/client/__init__.py @@ -134,7 +134,7 @@ class LocalClient(object): def __init__(self, c_path=os.path.join(syspaths.CONFIG_DIR, 'master'), mopts=None, skip_perm_errors=False, - io_loop=None): + io_loop=None, keep_loop=False): ''' :param IOLoop io_loop: io_loop used for events. Pass in an io_loop if you want asynchronous @@ -163,7 +163,8 @@ class LocalClient(object): self.opts['transport'], opts=self.opts, listen=False, - io_loop=io_loop) + io_loop=io_loop, + keep_loop=keep_loop) self.utils = salt.loader.utils(self.opts) self.functions = salt.loader.minion_mods(self.opts, utils=self.utils) self.returners = salt.loader.returners(self.opts, self.functions) diff --git a/salt/fileclient.py b/salt/fileclient.py index 7a58f13a30..3f6eed73c5 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -709,12 +709,17 @@ class Client(object): elif not os.path.isabs(cachedir): cachedir = os.path.join(self.opts['cachedir'], cachedir) + if url_data.query is not None: + file_name = '-'.join([url_data.path, url_data.query]) + else: + file_name = url_data.path + return salt.utils.path_join( cachedir, 'extrn_files', saltenv, netloc, - url_data.path + file_name ) diff --git a/salt/grains/core.py b/salt/grains/core.py index 4eb9bd2785..32cd34f4ef 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -1005,6 +1005,7 @@ _OS_NAME_MAP = { 'enterprise': 'OEL', 'oracleserv': 'OEL', 'cloudserve': 'CloudLinux', + 'cloudlinux': 'CloudLinux', 'pidora': 'Fedora', 'scientific': 'ScientificLinux', 'synology': 'Synology', @@ -1289,7 +1290,12 @@ def os_data(): grains[ 'lsb_{0}'.format(match.groups()[0].lower()) ] = match.groups()[1].rstrip() - if 'lsb_distrib_id' not in grains: + if grains.get('lsb_distrib_description', '').lower().startswith('antergos'): + # Antergos incorrectly configures their /etc/lsb-release, + # setting the DISTRIB_ID to "Arch". This causes the "os" grain + # to be incorrectly set to "Arch". + grains['osfullname'] = 'Antergos Linux' + elif 'lsb_distrib_id' not in grains: if os.path.isfile('/etc/os-release') or os.path.isfile('/usr/lib/os-release'): os_release = _parse_os_release() if 'NAME' in os_release: diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py index c5f935bc6f..9335d22eb8 100644 --- a/salt/modules/aptpkg.py +++ b/salt/modules/aptpkg.py @@ -1048,14 +1048,20 @@ def upgrade(refresh=True, dist_upgrade=False, **kwargs): force_conf = '--force-confnew' else: force_conf = '--force-confold' - cmd = [] if salt.utils.systemd.has_scope(__context__) \ and __salt__['config.get']('systemd.scope', True): cmd.extend(['systemd-run', '--scope']) + cmd.extend(['apt-get', '-q', '-y', '-o', 'DPkg::Options::={0}'.format(force_conf), '-o', 'DPkg::Options::=--force-confdef']) + + if kwargs.get('force_yes', False): + cmd.append('--force-yes') + if kwargs.get('skip_verify', False): + cmd.append('--allow-unauthenticated') + cmd.append('dist-upgrade' if dist_upgrade else 'upgrade') call = __salt__['cmd.run_all'](cmd, diff --git a/salt/modules/boto_iam.py b/salt/modules/boto_iam.py index 177cb74f79..35c605f61f 100644 --- a/salt/modules/boto_iam.py +++ b/salt/modules/boto_iam.py @@ -1299,6 +1299,7 @@ def get_account_id(region=None, key=None, keyid=None, profile=None): # The get_user call returns an user ARN: # arn:aws:iam::027050522557:user/salt-test arn = ret['get_user_response']['get_user_result']['user']['arn'] + account_id = arn.split(':')[4] except boto.exception.BotoServerError: # If call failed, then let's try to get the ARN from the metadata timeout = boto.config.getfloat( @@ -1307,15 +1308,15 @@ def get_account_id(region=None, key=None, keyid=None, profile=None): attempts = boto.config.getint( 'Boto', 'metadata_service_num_attempts', 1 ) - metadata = boto.utils.get_instance_metadata( + identity = boto.utils.get_instance_identity( timeout=timeout, num_retries=attempts ) try: - arn = metadata['iam']['info']['InstanceProfileArn'] + account_id = identity['document']['accountId'] except KeyError: - log.error('Failed to get user or metadata ARN information in' + log.error('Failed to get account id from instance_identity in' ' boto_iam.get_account_id.') - __context__[cache_key] = arn.split(':')[4] + __context__[cache_key] = account_id return __context__[cache_key] diff --git a/salt/modules/boto_sns.py b/salt/modules/boto_sns.py index 2b1311f950..fd511dbd2e 100644 --- a/salt/modules/boto_sns.py +++ b/salt/modules/boto_sns.py @@ -227,6 +227,9 @@ def get_arn(name, region=None, key=None, keyid=None, profile=None): def _get_region(region=None, profile=None): if profile and 'region' in profile: return profile['region'] + if not region and __salt__['config.option'](profile): + _profile = __salt__['config.option'](profile) + region = _profile.get('region', None) if not region and __salt__['config.option']('sns.region'): region = __salt__['config.option']('sns.region') if not region: diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index e9a5f8b9b3..e8baee782b 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -20,6 +20,7 @@ import time import traceback import fnmatch import base64 +import re # Import salt libs import salt.utils @@ -2643,6 +2644,151 @@ def shells(): return ret +def shell_info(shell): + ''' + Provides information about a shell or languages often use #! + The values returned are dependant on the shell or languages all return the + ``installed``, ``path``, ``version``, ``version_major`` and + ``version_major_minor`` e.g. 5.0 or 6-3. + The shell must be within the exeuctable search path. + + :param str shell: Name of the shell. + Support are bash, cmd, perl, php, powershell, python, ruby and zsh + :return: Properies of the shell specifically its and other information if + available. + :rtype: dict + + .. code-block:: cfg + {'version': '', + 'version_major': '', + 'path': '', + 'installed': , + '': ''} + + ``installed`` is always returned, if None or False also returns error and + may also return stdout for diagnostics. + + .. versionadded:: 2016.9.0 + + CLI Example:: + .. code-block:: bash + salt '*' cmd.shell bash + salt '*' cmd.shell powershell + + :codeauthor: Damon Atkins + ''' + regex_shells = { + 'bash': [r'version ([-\w.]+)', 'bash', '--version'], + 'bash-test-error': [r'versioZ ([-\w.]+)', 'bash', '--version'], # used to test a error result + 'bash-test-env': [r'(HOME=.*)', 'bash', '-c', 'declare'], # used to test a error result + 'zsh': [r'^zsh ([\d.]+)', 'zsh', '--version'], + 'tcsh': [r'^tcsh ([\d.]+)', 'tcsh', '--version'], + 'cmd': [r'Version ([\d.]+)', 'cmd.exe', '/C', 'ver'], + 'powershell': [r'PSVersion\s+([\d.]+)', 'powershell', '-NonInteractive', '$PSVersionTable'], + 'perl': [r'^([\d.]+)', 'perl', '-e', 'printf "%vd\n", $^V;'], + 'python': [r'^Python ([\d.]+)', 'python', '-V'], + 'ruby': [r'^ruby ([\d.]+)', 'ruby', '-v'], + 'php': [r'^PHP ([\d.]+)', 'php', '-v'] + } + # Ensure ret['installed'] always as a value of True, False or None (not sure) + ret = {} + ret['installed'] = None + if salt.utils.is_windows() and shell == 'powershell': + pw_keys = __salt__['reg.list_keys']('HKEY_LOCAL_MACHINE', 'Software\\Microsoft\\PowerShell') + pw_keys.sort(key=int) + if len(pw_keys) == 0: + return { + 'error': 'Unable to locate \'powershell\' Reason: Cannot be found in registry.', + 'installed': False, + 'version': None + } + for reg_ver in pw_keys: + install_data = __salt__['reg.read_value']('HKEY_LOCAL_MACHINE', 'Software\\Microsoft\\PowerShell\\{0}'.format(reg_ver), 'Install') + if 'vtype' in install_data and install_data['vtype'] == 'REG_DWORD' and install_data['vdata'] == 1: + details = __salt__['reg.list_values']('HKEY_LOCAL_MACHINE', 'Software\\Microsoft\\PowerShell\\{0}\\PowerShellEngine'.format(reg_ver)) + ret = {} # reset data, want the newest version details only as powershell is backwards compatible + ret['installed'] = True + ret['path'] = which('powershell.exe') + for attribute in details: + if attribute['vname'].lower() == '(default)': + continue + elif attribute['vname'].lower() == 'powershellversion': + ret['psversion'] = attribute['vdata'] + ret['version'] = attribute['vdata'] + elif attribute['vname'].lower() == 'runtimeversion': + ret['crlversion'] = attribute['vdata'] + if ret['crlversion'][0] == 'v' or ret['crlversion'][0] == 'V': + ret['crlversion'] = ret['crlversion'][1::] + elif attribute['vname'].lower() == 'pscompatibleversion': + # reg attribute does not end in s, the powershell attibute does + ret['pscompatibleversions'] = attribute['vdata'] + else: + # keys are lower case as python is case sensitive the registry is not + ret[attribute['vname'].lower()] = attribute['vdata'] + else: + if shell not in regex_shells: + return { + 'error': 'Salt does not know how to get the version number for {0}'.format(shell), + 'installed': None + } + shell_data = regex_shells[shell] + pattern = shell_data.pop(0) + # We need to make sure HOME set, so shells work correctly + # salt-call will general have home set, the salt-minion service may not + # We need to assume ports of unix shells to windows will look after themselves + # in setting HOME as they do it in many different ways + newenv = os.environ + if ('HOME' not in newenv) and (not salt.utils.is_windows()): + newenv['HOME'] = os.path.expanduser('~') + log.debug('HOME environment set to {0}'.format(newenv['HOME'])) + try: + proc = salt.utils.timed_subprocess.TimedProc( + shell_data, + stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=10, + env=newenv + ) + except (OSError, IOError) as exc: + return { + 'error': 'Unable to run command \'{0}\' Reason: {1}'.format(' '.join(shell_data), exc), + 'installed': False, + 'version': None + } + try: + proc.run() + except TimedProcTimeoutError as exc: + return { + 'error': 'Unable to run command \'{0}\' Reason: Timed out.'.format(' '.join(shell_data)), + 'installed': False, + 'version': None + } + + ret['installed'] = True + ret['path'] = which(shell_data[0]) + pattern_result = re.search(pattern, proc.stdout, flags=re.IGNORECASE) + # only set version if we find it, so code later on can deal with it + if pattern_result: + ret['version'] = pattern_result.group(1) + + if 'version' not in ret: + ret['error'] = 'The version regex pattern for shell {0}, could not find the version string'.format(shell) + ret['version'] = None + ret['stdout'] = proc.stdout # include stdout so they can see the issue + log.error(ret['error']) + else: + major_result = re.match('(\\d+)', ret['version']) + if major_result: + ret['version_major'] = major_result.group(1) + major_minor_result = re.match('(\\d+[-.]\\d+)', ret['version']) + if major_minor_result: + ret['version_major_minor'] = major_minor_result.group(1) + + return ret + + def powershell(cmd, cwd=None, stdin=None, diff --git a/salt/modules/pacman.py b/salt/modules/pacman.py index 6a4add2240..ec348876fe 100644 --- a/salt/modules/pacman.py +++ b/salt/modules/pacman.py @@ -36,7 +36,7 @@ def __virtual__(): ''' Set the virtual pkg module if the os is Arch ''' - if __grains__['os'] in ('Arch', 'Arch ARM', 'Antergos', 'ManjaroLinux'): + if __grains__['os_family'] == 'Arch': return __virtualname__ return (False, 'The pacman module could not be loaded: unsupported OS family.') @@ -155,10 +155,17 @@ def list_upgrades(refresh=False, root=None, **kwargs): # pylint: disable=W0613 out = call['stdout'] for line in salt.utils.itertools.split(out, '\n'): - comps = line.split(' ') - if len(comps) != 2: + try: + pkgname, pkgver = line.split() + except ValueError: continue - upgrades[comps[0]] = comps[1] + if pkgname.lower() == 'downloading' and '.db' in pkgver.lower(): + # Antergos (and possibly other Arch derivatives) add lines when pkg + # metadata is being downloaded. Because these lines, when split, + # contain two columns (i.e. 'downloading community.db...'), we will + # skip this line to keep it from being interpreted as an upgrade. + continue + upgrades[pkgname] = pkgver return upgrades diff --git a/salt/modules/win_dsc.py b/salt/modules/win_dsc.py index 873d816b49..9a74c2f8e9 100644 --- a/salt/modules/win_dsc.py +++ b/salt/modules/win_dsc.py @@ -7,13 +7,14 @@ Module for managing PowerShell modules Support for PowerShell ''' -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals # Import python libs import copy import logging import json import os +import distutils.version # Import salt libs import salt.utils @@ -33,25 +34,39 @@ def __virtual__(): if not salt.utils.is_windows(): return (False, 'Module DSC: Module only works on Windows systems ') - if psversion() < 5: - return (False, 'Module DSC: Module only works with PowerShell 5 or later.') + powershell_info = __salt__['cmd.shell_info']('powershell') + if ( + powershell_info['installed'] and + ('version_major' in powershell_info) and + (distutils.version.LooseVersion(powershell_info['version_major']) < distutils.version.LooseVersion('5')) + ): + return (False, 'Module DSC: Module only works with PowerShell 5 or newer.') return __virtualname__ -def _pshell(cmd, cwd=None): +def _pshell(cmd, cwd=None, json_depth=2): ''' Execute the desired powershell command and ensure that it returns data in json format and load that into python ''' if 'convertto-json' not in cmd.lower(): - cmd = ' '.join([cmd, '| ConvertTo-Json']) + cmd = '{0} | ConvertTo-Json -Depth {1}'.format(cmd, json_depth) log.debug('DSC: {0}'.format(cmd)) - ret = __salt__['cmd.shell'](cmd, shell='powershell', cwd=cwd) + results = __salt__['cmd.run_all'](cmd, shell='powershell', cwd=cwd, python_shell=True) + + if 'pid' in results: + del results['pid'] + + if 'retcode' not in results or results['retcode'] != 0: + # run_all logs an error to log.error, fail hard back to the user + raise CommandExecutionError('Issue executing powershell {0}'.format(cmd), info=results) + try: - ret = json.loads(ret, strict=False) + ret = json.loads(results['stdout'], strict=False) except ValueError: - log.debug('Json not returned') + raise CommandExecutionError('No JSON results from powershell', info=results) + return ret @@ -92,15 +107,27 @@ def psversion(): ''' Returns the Powershell version + This has been deprecated and has been replaced by ``cmd.shell_info`` Note + the minimum version return is 5 as ``dsc`` is not available for version + less than 5. This function will be removed in 'Nitrogen' release. + CLI Example: .. code-block:: bash salt 'win01' dsc.psversion ''' - cmd = '$PSVersionTable.PSVersion.Major' - ret = _pshell(cmd) - return ret + salt.utils.warn_until('Nitrogen', + 'The \'psversion\' has been deprecated and has been ' + 'replaced by \'cmd.shell_info\'.' + ) + powershell_info = __salt__['cmd.shell_info']('powershell') + if powershell_info['installed'] and 'version_major' in powershell_info: + try: + return int(powershell_info['version_major']) + except ValueError: + pass + return 0 def avail_modules(desc=False): diff --git a/salt/modules/win_psget.py b/salt/modules/win_psget.py index 88ca814377..367100e6d8 100644 --- a/salt/modules/win_psget.py +++ b/salt/modules/win_psget.py @@ -8,15 +8,17 @@ Module for managing PowerShell through PowerShellGet (PSGet) Support for PowerShell ''' -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals # Import python libs import copy import logging import json +import distutils.version # Import salt libs import salt.utils +from salt.exceptions import CommandExecutionError # Set up logging log = logging.getLogger(__name__) @@ -32,25 +34,39 @@ def __virtual__(): if not salt.utils.is_windows(): return (False, 'Module PSGet: Module only works on Windows systems ') - if psversion() < 5: - return (False, 'Module PSGet: Module only works with PowerShell 5 or later.') + powershell_info = __salt__['cmd.shell_info']('powershell') + if ( + powershell_info['installed'] and + ('version_major' in powershell_info) and + (distutils.version.LooseVersion(powershell_info['version_major']) < distutils.version.LooseVersion('5')) + ): + return (False, 'Module PSGet: Module only works with PowerShell 5 or newer.') return __virtualname__ -def _pshell(cmd, cwd=None): +def _pshell(cmd, cwd=None, json_depth=2): ''' Execute the desired powershell command and ensure that it returns data in json format and load that into python ''' if 'convertto-json' not in cmd.lower(): - cmd = ' '.join([cmd, '| ConvertTo-Json']) - log.debug('PSGET: {0}'.format(cmd)) - ret = __salt__['cmd.shell'](cmd, shell='powershell', cwd=cwd) + cmd = '{0} | ConvertTo-Json -Depth {1}'.format(cmd, json_depth) + log.debug('DSC: {0}'.format(cmd)) + results = __salt__['cmd.run_all'](cmd, shell='powershell', cwd=cwd, python_shell=True) + + if 'pid' in results: + del results['pid'] + + if 'retcode' not in results or results['retcode'] != 0: + # run_all logs an error to log.error, fail hard back to the user + raise CommandExecutionError('Issue executing powershell {0}'.format(cmd), info=results) + try: - ret = json.loads(ret, strict=False) + ret = json.loads(results['stdout'], strict=False) except ValueError: - log.debug('Json not returned') + raise CommandExecutionError('No JSON results from powershell', info=results) + return ret @@ -58,15 +74,27 @@ def psversion(): ''' Returns the Powershell version + This has been deprecated and has been replaced by ``cmd.shell_info`` Note + the minimum version return is 5 as ``dsc`` is not available for version + less than 5. This function will be removed in 'Nitrogen' release. + CLI Example: .. code-block:: bash salt 'win01' dsc.psversion ''' - cmd = '$PSVersionTable.PSVersion.Major' - ret = _pshell(cmd) - return ret + salt.utils.warn_until('Nitrogen', + 'The \'psversion\' has been deprecated and has been ' + 'replaced by \'cmd.shell_info\'.' + ) + powershell_info = __salt__['cmd.shell_info']('powershell') + if powershell_info['installed'] and 'version_major' in powershell_info: + try: + return int(powershell_info['version_major']) + except ValueError: + pass + return 0 def bootstrap(): diff --git a/salt/modules/win_service.py b/salt/modules/win_service.py index c21a1c5505..1b190ff8f5 100644 --- a/salt/modules/win_service.py +++ b/salt/modules/win_service.py @@ -406,7 +406,13 @@ def stop(name): salt '*' service.stop ''' - if not status(name): + # net stop issues a stop command and waits briefly (~30s), but will give + # up if the service takes too long to stop with a misleading + # "service could not be stopped" message and RC 0. + + cmd = ['net', 'stop', '/y', name] + res = __salt__['cmd.run'](cmd, python_shell=False) + if 'service was stopped' in res: return True try: diff --git a/salt/netapi/rest_cherrypy/app.py b/salt/netapi/rest_cherrypy/app.py index 7df594b316..d267f46543 100644 --- a/salt/netapi/rest_cherrypy/app.py +++ b/salt/netapi/rest_cherrypy/app.py @@ -900,7 +900,7 @@ def lowdata_fmt(): # if the data was sent as urlencoded, we need to make it a list. # this is a very forgiving implementation as different clients set different # headers for form encoded data (including charset or something similar) - if data and not isinstance(data, list): + if data and isinstance(data, collections.Mapping): # Make the 'arg' param a list if not already if 'arg' in data and not isinstance(data['arg'], list): data['arg'] = [data['arg']] diff --git a/salt/state.py b/salt/state.py index 3dfcdc2c0f..a540bcf1ea 100644 --- a/salt/state.py +++ b/salt/state.py @@ -1832,6 +1832,8 @@ class State(object): Check if the low data chunk should send a failhard signal ''' tag = _gen_tag(low) + if self.opts['test']: + return False if (low.get('failhard', False) or self.opts['failhard'] and tag in running): if running[tag]['result'] is None: diff --git a/salt/states/git.py b/salt/states/git.py index a9717770b6..ef3ff2577e 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -1158,6 +1158,20 @@ def latest(name, else: branch_opts = None + if branch_opts is not None and local_branch is None: + return _fail( + ret, + 'Cannot set/unset upstream tracking branch, local ' + 'HEAD refers to nonexistent branch. This may have ' + 'been caused by cloning a remote repository for which ' + 'the default branch was renamed or deleted. If you ' + 'are unable to fix the remote repository, you can ' + 'work around this by setting the \'branch\' argument ' + '(which will ensure that the named branch is created ' + 'if it does not already exist).', + comments + ) + if not has_remote_rev: try: fetch_changes = __salt__['git.fetch']( @@ -1577,6 +1591,21 @@ def latest(name, local_rev, local_branch = \ _get_local_rev_and_branch(target, user, password) + if local_branch is None \ + and remote_rev is not None \ + and 'HEAD' not in all_remote_refs: + return _fail( + ret, + 'Remote HEAD refers to a ref that does not exist. ' + 'This can happen when the default branch on the ' + 'remote repository is renamed or deleted. If you ' + 'are unable to fix the remote repository, you can ' + 'work around this by setting the \'branch\' argument ' + '(which will ensure that the named branch is created ' + 'if it does not already exist).', + comments + ) + if not _revs_equal(local_rev, remote_rev, remote_rev_type): __salt__['git.reset']( target, diff --git a/salt/utils/event.py b/salt/utils/event.py index 4a59ebacd9..70b3c423f1 100644 --- a/salt/utils/event.py +++ b/salt/utils/event.py @@ -111,7 +111,7 @@ TAGS = { def get_event( node, sock_dir=None, transport='zeromq', - opts=None, listen=True, io_loop=None): + opts=None, listen=True, io_loop=None, keep_loop=False): ''' Return an event object suitable for the named transport @@ -124,8 +124,8 @@ def get_event( # TODO: AIO core is separate from transport if transport in ('zeromq', 'tcp'): if node == 'master': - return MasterEvent(sock_dir, opts, listen=listen, io_loop=io_loop) - return SaltEvent(node, sock_dir, opts, listen=listen, io_loop=io_loop) + return MasterEvent(sock_dir, opts, listen=listen, io_loop=io_loop, keep_loop=keep_loop) + return SaltEvent(node, sock_dir, opts, listen=listen, io_loop=io_loop, keep_loop=keep_loop) elif transport == 'raet': import salt.utils.raetevent return salt.utils.raetevent.RAETEvent(node, @@ -197,14 +197,19 @@ class SaltEvent(object): ''' def __init__( self, node, sock_dir=None, - opts=None, listen=True, io_loop=None): + opts=None, listen=True, io_loop=None, keep_loop=False): ''' :param IOLoop io_loop: Pass in an io_loop if you want asynchronous operation for obtaining events. Eg use of set_event_handler() API. Otherwise, operation will be synchronous. + :param Bool keep_loop: Pass a boolean to determine if we want to keep + the io loop or destroy it when the event handle + is destroyed. This is useful when using event + loops from within third party async code ''' self.serial = salt.payload.Serial({'serial': 'msgpack'}) + self.keep_loop = keep_loop if io_loop is not None: self.io_loop = io_loop self._run_io_loop_sync = False @@ -727,7 +732,7 @@ class SaltEvent(object): self.subscriber.close() if self.pusher is not None: self.pusher.close() - if self._run_io_loop_sync: + if self._run_io_loop_sync and not self.keep_loop: self.io_loop.close() def fire_ret_load(self, load): @@ -790,9 +795,20 @@ class MasterEvent(SaltEvent): RAET compatible Create a master event management object ''' - def __init__(self, sock_dir, opts=None, listen=True, io_loop=None): + def __init__( + self, + sock_dir, + opts=None, + listen=True, + io_loop=None, + keep_loop=False): super(MasterEvent, self).__init__( - 'master', sock_dir, opts, listen=listen, io_loop=io_loop) + 'master', + sock_dir, + opts, + listen=listen, + io_loop=io_loop, + keep_loop=keep_loop) class LocalClientEvent(MasterEvent): diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index 48028835c6..d5218f34d9 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -449,7 +449,7 @@ class Schedule(object): with salt.utils.fopen(schedule_conf, 'wb+') as fp_: fp_.write( salt.utils.to_bytes( - yaml.dump({'schedule': self.opts['schedule']}) + yaml.dump({'schedule': self.option('schedule')}) ) ) except (IOError, OSError): @@ -462,9 +462,9 @@ class Schedule(object): ''' if where is None or where != 'pillar': # ensure job exists, then delete it - if name in self.opts['schedule']: - del self.opts['schedule'][name] - schedule = self.opts['schedule'] + schedule = self.option('schedule') + if name in schedule: + del schedule[name] else: # If job is in pillar, delete it there too if 'schedule' in self.opts['pillar']: @@ -491,10 +491,10 @@ class Schedule(object): ''' if where is None or where != 'pillar': # ensure job exists, then delete it - for job in list(self.opts['schedule'].keys()): + schedule = self.option('schedule') + for job in list(schedule.keys()): if job.startswith(name): - del self.opts['schedule'][job] - schedule = self.opts['schedule'] + del schedule[job] else: # If job is in pillar, delete it there too if 'schedule' in self.opts['pillar']: @@ -539,17 +539,18 @@ class Schedule(object): new_job = next(six.iterkeys(data)) - if new_job in self.opts['schedule']: + schedule = self.option('schedule') + if new_job in schedule: log.info('Updating job settings for scheduled ' 'job: {0}'.format(new_job)) else: log.info('Added new job {0} to scheduler'.format(new_job)) - self.opts['schedule'].update(data) + schedule.update(data) # Fire the complete event back along with updated list of schedule evt = salt.utils.event.get_event('minion', opts=self.opts, listen=False) - evt.fire_event({'complete': True, 'schedule': self.opts['schedule']}, + evt.fire_event({'complete': True, 'schedule': schedule}, tag='/salt/minion/minion_schedule_add_complete') if persist: @@ -563,8 +564,8 @@ class Schedule(object): self.opts['pillar']['schedule'][name]['enabled'] = True schedule = self.opts['pillar']['schedule'] else: - self.opts['schedule'][name]['enabled'] = True - schedule = self.opts['schedule'] + schedule = self.option('schedule') + schedule[name]['enabled'] = True # Fire the complete event back along with updated list of schedule evt = salt.utils.event.get_event('minion', opts=self.opts, listen=False) @@ -584,8 +585,8 @@ class Schedule(object): self.opts['pillar']['schedule'][name]['enabled'] = False schedule = self.opts['pillar']['schedule'] else: - self.opts['schedule'][name]['enabled'] = False - schedule = self.opts['schedule'] + schedule = self.option('schedule') + schedule[name]['enabled'] = False # Fire the complete event back along with updated list of schedule evt = salt.utils.event.get_event('minion', opts=self.opts, listen=False) @@ -606,9 +607,9 @@ class Schedule(object): self.delete_job(name, persist, where=where) self.opts['pillar']['schedule'][name] = schedule else: - if name in self.opts['schedule']: + if name in self.option('schedule'): self.delete_job(name, persist, where=where) - self.opts['schedule'][name] = schedule + self.option('schedule')[name] = schedule if persist: self.persist() @@ -617,7 +618,7 @@ class Schedule(object): ''' Run a schedule job now ''' - schedule = self.opts['schedule'] + schedule = self.option('schedule') if 'schedule' in self.opts['pillar']: schedule.update(self.opts['pillar']['schedule']) data = schedule[name] @@ -664,22 +665,24 @@ class Schedule(object): ''' Enable the scheduler. ''' - self.opts['schedule']['enabled'] = True + schedule = self.option('schedule') + schedule['enabled'] = True # Fire the complete event back along with updated list of schedule evt = salt.utils.event.get_event('minion', opts=self.opts, listen=False) - evt.fire_event({'complete': True, 'schedule': self.opts['schedule']}, + evt.fire_event({'complete': True, 'schedule': schedule}, tag='/salt/minion/minion_schedule_enabled_complete') def disable_schedule(self): ''' Disable the scheduler. ''' - self.opts['schedule']['enabled'] = False + schedule = self.option('schedule') + schedule['enabled'] = False # Fire the complete event back along with updated list of schedule evt = salt.utils.event.get_event('minion', opts=self.opts, listen=False) - evt.fire_event({'complete': True, 'schedule': self.opts['schedule']}, + evt.fire_event({'complete': True, 'schedule': schedule}, tag='/salt/minion/minion_schedule_disabled_complete') def reload(self, schedule): @@ -702,9 +705,9 @@ class Schedule(object): if 'schedule' in self.opts['pillar']: schedule.update(self.opts['pillar']['schedule']) elif where == 'opts': - schedule.update(self.opts['schedule']) + schedule.update(self.option('schedule')) else: - schedule.update(self.opts['schedule']) + schedule.update(self.option('schedule')) if 'schedule' in self.opts['pillar']: schedule.update(self.opts['pillar']['schedule']) diff --git a/setup.py b/setup.py index 24e66536e2..54c4210704 100755 --- a/setup.py +++ b/setup.py @@ -119,6 +119,7 @@ SALT_VERSION = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', 'version.py' SALT_VERSION_HARDCODED = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', '_version.py') SALT_SYSPATHS_HARDCODED = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', '_syspaths.py') SALT_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'base.txt') +SALT_WINDOWS_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'windows.txt') SALT_ZEROMQ_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'zeromq.txt') SALT_RAET_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'raet.txt') @@ -384,11 +385,11 @@ class InstallPyCryptoWindowsWheel(Command): call_arguments = ['pip', 'install', 'wheel'] if platform_bits == '64bit': call_arguments.append( - 'http://repo.saltstack.com/windows/dependencies/64/pycrypto-2.6.1-cp27-none-win_amd64.whl' + 'https://repo.saltstack.com/windows/dependencies/64/pycrypto-2.6.1-cp27-none-win_amd64.whl' ) else: call_arguments.append( - 'http://repo.saltstack.com/windows/dependencies/32/pycrypto-2.6.1-cp27-none-win32.whl' + 'https://repo.saltstack.com/windows/dependencies/32/pycrypto-2.6.1-cp27-none-win32.whl' ) with indent_log(): call_subprocess(call_arguments) @@ -415,11 +416,11 @@ class InstallCompiledPyYaml(Command): call_arguments = ['easy_install', '-Z'] if platform_bits == '64bit': call_arguments.append( - 'http://repo.saltstack.com/windows/dependencies/64/PyYAML-3.11.win-amd64-py2.7.exe' + 'https://repo.saltstack.com/windows/dependencies/64/PyYAML-3.11.win-amd64-py2.7.exe' ) else: call_arguments.append( - 'http://repo.saltstack.com/windows/dependencies/32/PyYAML-3.11.win-amd64-py2.7.exe' + 'https://repo.saltstack.com/windows/dependencies/32/PyYAML-3.11.win32-py2.7.exe' ) with indent_log(): call_subprocess(call_arguments) @@ -442,7 +443,7 @@ class DownloadWindowsDlls(Command): import platform from pip.utils.logging import indent_log platform_bits, _ = platform.architecture() - url = 'http://repo.saltstack.com/windows/dependencies/{bits}/{fname}.dll' + url = 'https://repo.saltstack.com/windows/dependencies/{bits}/{fname}.dll' dest = os.path.join(os.path.dirname(sys.executable), '{fname}.dll') with indent_log(): for fname in ('libeay32', 'ssleay32', 'libsodium', 'msvcr120'): @@ -1023,8 +1024,7 @@ class SaltDistribution(distutils.dist.Distribution): install_requires = _parse_requirements_file(SALT_REQS) if IS_WINDOWS_PLATFORM: - install_requires.append('WMI') - install_requires.append('pypiwin32 >= 219') + install_requires += _parse_requirements_file(SALT_WINDOWS_REQS) if self.salt_transport == 'zeromq': install_requires += _parse_requirements_file(SALT_ZEROMQ_REQS) diff --git a/tests/integration/states/git.py b/tests/integration/states/git.py index 7945b1cb2d..ba58fa2288 100644 --- a/tests/integration/states/git.py +++ b/tests/integration/states/git.py @@ -8,6 +8,7 @@ from __future__ import absolute_import import os import shutil import socket +import string import subprocess import tempfile @@ -328,6 +329,136 @@ class GitTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn): self.assertSaltTrueReturn(ret) +@skip_if_binaries_missing('git') +class LocalRepoGitTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn): + ''' + Tests which do no require connectivity to github.com + ''' + def test_renamed_default_branch(self): + ''' + Test the case where the remote branch has been removed + https://github.com/saltstack/salt/issues/36242 + ''' + cwd = os.getcwd() + repo = tempfile.mkdtemp(dir=integration.TMP) + admin = tempfile.mkdtemp(dir=integration.TMP) + name = tempfile.mkdtemp(dir=integration.TMP) + for dirname in (repo, admin, name): + self.addCleanup(shutil.rmtree, dirname, ignore_errors=True) + self.addCleanup(os.chdir, cwd) + + with salt.utils.fopen(os.devnull, 'w') as devnull: + # Create bare repo + subprocess.check_call(['git', 'init', '--bare', repo], + stdout=devnull, stderr=devnull) + # Clone bare repo + subprocess.check_call(['git', 'clone', repo, admin], + stdout=devnull, stderr=devnull) + + # Create, add, commit, and push file + os.chdir(admin) + with salt.utils.fopen('foo', 'w'): + pass + subprocess.check_call(['git', 'add', '.'], + stdout=devnull, stderr=devnull) + subprocess.check_call(['git', 'commit', '-m', 'init'], + stdout=devnull, stderr=devnull) + subprocess.check_call(['git', 'push', 'origin', 'master'], + stdout=devnull, stderr=devnull) + + # Change back to the original cwd + os.chdir(cwd) + + # Rename remote 'master' branch to 'develop' + os.rename( + os.path.join(repo, 'refs', 'heads', 'master'), + os.path.join(repo, 'refs', 'heads', 'develop') + ) + + # Run git.latest state. This should successfuly clone and fail with a + # specific error in the comment field. + ret = self.run_state( + 'git.latest', + name=repo, + target=name, + rev='develop', + ) + self.assertSaltFalseReturn(ret) + self.assertEqual( + ret[next(iter(ret))]['comment'], + 'Remote HEAD refers to a ref that does not exist. ' + 'This can happen when the default branch on the ' + 'remote repository is renamed or deleted. If you ' + 'are unable to fix the remote repository, you can ' + 'work around this by setting the \'branch\' argument ' + '(which will ensure that the named branch is created ' + 'if it does not already exist).\n\n' + 'Changes already made: {0} cloned to {1}' + .format(repo, name) + ) + self.assertEqual( + ret[next(iter(ret))]['changes'], + {'new': '{0} => {1}'.format(repo, name)} + ) + + # Run git.latest state again. This should fail again, with a different + # error in the comment field, and should not change anything. + ret = self.run_state( + 'git.latest', + name=repo, + target=name, + rev='develop', + ) + self.assertSaltFalseReturn(ret) + self.assertEqual( + ret[next(iter(ret))]['comment'], + 'Cannot set/unset upstream tracking branch, local ' + 'HEAD refers to nonexistent branch. This may have ' + 'been caused by cloning a remote repository for which ' + 'the default branch was renamed or deleted. If you ' + 'are unable to fix the remote repository, you can ' + 'work around this by setting the \'branch\' argument ' + '(which will ensure that the named branch is created ' + 'if it does not already exist).' + ) + self.assertEqual(ret[next(iter(ret))]['changes'], {}) + + # Run git.latest state again with a branch manually set. This should + # checkout a new branch and the state should pass. + ret = self.run_state( + 'git.latest', + name=repo, + target=name, + rev='develop', + branch='develop', + ) + # State should succeed + self.assertSaltTrueReturn(ret) + self.assertSaltCommentRegexpMatches( + ret, + 'New branch \'develop\' was checked out, with origin/develop ' + r'\([0-9a-f]{7}\) as a starting point' + ) + # Only the revision should be in the changes dict. + self.assertEqual( + list(ret[next(iter(ret))]['changes'].keys()), + ['revision'] + ) + # Since the remote repo was incorrectly set up, the local head should + # not exist (therefore the old revision should be None). + self.assertEqual( + ret[next(iter(ret))]['changes']['revision']['old'], + None + ) + # Make sure the new revision is a SHA (40 chars, all hex) + self.assertTrue( + len(ret[next(iter(ret))]['changes']['revision']['new']) == 40) + self.assertTrue( + all([x in string.hexdigits for x in + ret[next(iter(ret))]['changes']['revision']['new']]) + ) + + if __name__ == '__main__': from integration import run_tests run_tests(GitTest)