From e5371ac7202b84c721f17681c66121be28f0ea0b Mon Sep 17 00:00:00 2001 From: puneet kandhari Date: Wed, 14 Sep 2016 18:45:05 -0500 Subject: [PATCH 1/9] No force_yes parameter to pkg.upgrade #21248 We have a case for force_yes and skip_verify when upgrade --- salt/modules/aptpkg.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py index d3db2b1a3e..e1d262990d 100644 --- a/salt/modules/aptpkg.py +++ b/salt/modules/aptpkg.py @@ -949,9 +949,17 @@ def upgrade(refresh=True, dist_upgrade=False, **kwargs): if salt.utils.systemd.has_scope(__context__) \ and __salt__['config.get']('systemd.scope', True): cmd.extend(['systemd-run', '--scope']) + cmd.extend(['apt-get', '-q', '-y', '-o', 'DPkg::Options::={0}'.format(force_conf), '-o', 'DPkg::Options::=--force-confdef']) + + if kwargs.get('force_yes', False): + cmd.append('--force-yes') + + if kwargs.get('skip_verify', False): + cmd.append('--allow-unauthenticated') + cmd.append('dist-upgrade' if dist_upgrade else 'upgrade') call = __salt__['cmd.run_all'](cmd, From cbe98d97a31c0d2e241d5eb70fc4bf9a52fa9247 Mon Sep 17 00:00:00 2001 From: puneet kandhari Date: Thu, 15 Sep 2016 11:48:45 -0500 Subject: [PATCH 2/9] Fix pylint whitespace errors --- salt/modules/aptpkg.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py index e1d262990d..2b884ba9dd 100644 --- a/salt/modules/aptpkg.py +++ b/salt/modules/aptpkg.py @@ -944,7 +944,6 @@ def upgrade(refresh=True, dist_upgrade=False, **kwargs): force_conf = '--force-confnew' else: force_conf = '--force-confold' - cmd = [] if salt.utils.systemd.has_scope(__context__) \ and __salt__['config.get']('systemd.scope', True): @@ -952,14 +951,13 @@ def upgrade(refresh=True, dist_upgrade=False, **kwargs): cmd.extend(['apt-get', '-q', '-y', '-o', 'DPkg::Options::={0}'.format(force_conf), - '-o', 'DPkg::Options::=--force-confdef']) - + '-o', 'DPkg::Options::=--force-confdef'] + if kwargs.get('force_yes', False): cmd.append('--force-yes') - if kwargs.get('skip_verify', False): cmd.append('--allow-unauthenticated') - + cmd.append('dist-upgrade' if dist_upgrade else 'upgrade') call = __salt__['cmd.run_all'](cmd, From 892cc4cd48a52cc0ebddbadf6316508e155d3e2a Mon Sep 17 00:00:00 2001 From: puneet kandhari Date: Thu, 15 Sep 2016 12:48:58 -0500 Subject: [PATCH 3/9] Update aptpkg.py --- salt/modules/aptpkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py index 2b884ba9dd..60813a1e3c 100644 --- a/salt/modules/aptpkg.py +++ b/salt/modules/aptpkg.py @@ -951,7 +951,7 @@ def upgrade(refresh=True, dist_upgrade=False, **kwargs): cmd.extend(['apt-get', '-q', '-y', '-o', 'DPkg::Options::={0}'.format(force_conf), - '-o', 'DPkg::Options::=--force-confdef'] + '-o', 'DPkg::Options::=--force-confdef']) if kwargs.get('force_yes', False): cmd.append('--force-yes') From 3ac308ac76ff98e07459b5b803479296aac45557 Mon Sep 17 00:00:00 2001 From: puneet kandhari Date: Fri, 16 Sep 2016 11:22:11 -0500 Subject: [PATCH 4/9] Update aptpkg.py --- salt/modules/aptpkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py index 60813a1e3c..6f62535d34 100644 --- a/salt/modules/aptpkg.py +++ b/salt/modules/aptpkg.py @@ -948,7 +948,7 @@ def upgrade(refresh=True, dist_upgrade=False, **kwargs): if salt.utils.systemd.has_scope(__context__) \ and __salt__['config.get']('systemd.scope', True): cmd.extend(['systemd-run', '--scope']) - + cmd.extend(['apt-get', '-q', '-y', '-o', 'DPkg::Options::={0}'.format(force_conf), '-o', 'DPkg::Options::=--force-confdef']) From 04edea5c59019c87f68cf38a0e8951df5526ea0c Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 16 Sep 2016 15:07:26 -0600 Subject: [PATCH 5/9] Add '/y' switch to the net stop and start commands --- salt/modules/win_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/win_service.py b/salt/modules/win_service.py index 65fd405d10..1fb1d35c85 100644 --- a/salt/modules/win_service.py +++ b/salt/modules/win_service.py @@ -165,7 +165,7 @@ def start(name): salt '*' service.start ''' - cmd = ['net', 'start', name] + cmd = ['net', 'start', '/y', name] return not __salt__['cmd.retcode'](cmd, python_shell=False) @@ -183,7 +183,7 @@ def stop(name): # up if the service takes too long to stop with a misleading # "service could not be stopped" message and RC 0. - cmd = ['net', 'stop', name] + cmd = ['net', 'stop', '/y', name] res = __salt__['cmd.run'](cmd, python_shell=False) if 'service was stopped' in res: return True From a817aef1c2e0dd86bef264b3ec92dfcec78cedb4 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 16 Sep 2016 15:43:36 -0600 Subject: [PATCH 6/9] Add windows requirements file --- requirements/windows.txt | 10 ++++++++++ setup.py | 14 +++++++------- 2 files changed, 17 insertions(+), 7 deletions(-) create mode 100644 requirements/windows.txt diff --git a/requirements/windows.txt b/requirements/windows.txt new file mode 100644 index 0000000000..ddbe31dc27 --- /dev/null +++ b/requirements/windows.txt @@ -0,0 +1,10 @@ +backports-abc +backports.ssl-match-hostname +certifi +psutil +python-dateutil +pypiwin32 +pyzmq +six +timelib +WMI diff --git a/setup.py b/setup.py index 3668c55df8..4001b12047 100755 --- a/setup.py +++ b/setup.py @@ -111,6 +111,7 @@ SALT_VERSION = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', 'version.py' SALT_VERSION_HARDCODED = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', '_version.py') SALT_SYSPATHS_HARDCODED = os.path.join(os.path.abspath(SETUP_DIRNAME), 'salt', '_syspaths.py') SALT_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'base.txt') +SALT_WINDOWS_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'windows.txt') SALT_ZEROMQ_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'zeromq.txt') SALT_RAET_REQS = os.path.join(os.path.abspath(SETUP_DIRNAME), 'requirements', 'raet.txt') @@ -371,11 +372,11 @@ class InstallPyCryptoWindowsWheel(Command): call_arguments = ['pip', 'install', 'wheel'] if platform_bits == '64bit': call_arguments.append( - 'http://repo.saltstack.com/windows/dependencies/64/pycrypto-2.6.1-cp27-none-win_amd64.whl' + 'https://repo.saltstack.com/windows/dependencies/64/pycrypto-2.6.1-cp27-none-win_amd64.whl' ) else: call_arguments.append( - 'http://repo.saltstack.com/windows/dependencies/32/pycrypto-2.6.1-cp27-none-win32.whl' + 'https://repo.saltstack.com/windows/dependencies/32/pycrypto-2.6.1-cp27-none-win32.whl' ) with indent_log(): call_subprocess(call_arguments) @@ -402,11 +403,11 @@ class InstallCompiledPyYaml(Command): call_arguments = ['easy_install', '-Z'] if platform_bits == '64bit': call_arguments.append( - 'http://repo.saltstack.com/windows/dependencies/64/PyYAML-3.11.win-amd64-py2.7.exe' + 'https://repo.saltstack.com/windows/dependencies/64/PyYAML-3.11.win-amd64-py2.7.exe' ) else: call_arguments.append( - 'http://repo.saltstack.com/windows/dependencies/32/PyYAML-3.11.win-amd64-py2.7.exe' + 'https://repo.saltstack.com/windows/dependencies/32/PyYAML-3.11.win32-py2.7.exe' ) with indent_log(): call_subprocess(call_arguments) @@ -429,7 +430,7 @@ class DownloadWindowsDlls(Command): import platform from pip.utils.logging import indent_log platform_bits, _ = platform.architecture() - url = 'http://repo.saltstack.com/windows/dependencies/{bits}/{fname}.dll' + url = 'https://repo.saltstack.com/windows/dependencies/{bits}/{fname}.dll' dest = os.path.join(os.path.dirname(sys.executable), '{fname}.dll') with indent_log(): for fname in ('libeay32', 'ssleay32', 'libsodium', 'msvcr120'): @@ -1049,8 +1050,7 @@ class SaltDistribution(distutils.dist.Distribution): install_requires = _parse_requirements_file(SALT_REQS) if IS_WINDOWS_PLATFORM: - install_requires.append('WMI') - install_requires.append('pypiwin32 >= 219') + install_requires += _parse_requirements_file(SALT_WINDOWS_REQS) if self.salt_transport == 'zeromq': install_requires += _parse_requirements_file(SALT_ZEROMQ_REQS) From bb4d69f58a4fefcc0bc60932756c0a98d82e6484 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 19 Sep 2016 10:33:00 -0500 Subject: [PATCH 7/9] git.latest: fail gracefully for misconfigured remote repo (#36391) * git.latest: fail gracefully for misconfigured remote repo When the remote repo's HEAD refers to a nonexistent ref, this was causing a traceback when we tried to check if the upstream tracking branch needed to be changed after cloning the repo. This commit fixes this traceback by gracefully failing the state when the remote HEAD is not present in the ``git ls-remote`` output, but the desired remote revision doesn't exist. Additionally, a similar graceful failure now happens if the state is run again after we gracefully fail the first time, and we need to set the tracking branch. Trying to set the tracking branch when there is no local branch would fail with an ambiguous error like "fatal: branch 'master' does not exist", so before we even attempt to set the tracking branch, the state is failed with a more descriptive comment. * Add integration test for #36242 --- salt/states/git.py | 29 +++++++ tests/integration/states/git.py | 131 ++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) diff --git a/salt/states/git.py b/salt/states/git.py index ca7f8b749a..52e1d3c6a2 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -1083,6 +1083,20 @@ def latest(name, else: branch_opts = None + if branch_opts is not None and local_branch is None: + return _fail( + ret, + 'Cannot set/unset upstream tracking branch, local ' + 'HEAD refers to nonexistent branch. This may have ' + 'been caused by cloning a remote repository for which ' + 'the default branch was renamed or deleted. If you ' + 'are unable to fix the remote repository, you can ' + 'work around this by setting the \'branch\' argument ' + '(which will ensure that the named branch is created ' + 'if it does not already exist).', + comments + ) + if not has_remote_rev: try: fetch_changes = __salt__['git.fetch']( @@ -1482,6 +1496,21 @@ def latest(name, local_rev, local_branch = \ _get_local_rev_and_branch(target, user) + if local_branch is None \ + and remote_rev is not None \ + and 'HEAD' not in all_remote_refs: + return _fail( + ret, + 'Remote HEAD refers to a ref that does not exist. ' + 'This can happen when the default branch on the ' + 'remote repository is renamed or deleted. If you ' + 'are unable to fix the remote repository, you can ' + 'work around this by setting the \'branch\' argument ' + '(which will ensure that the named branch is created ' + 'if it does not already exist).', + comments + ) + if not _revs_equal(local_rev, remote_rev, remote_rev_type): __salt__['git.reset']( target, diff --git a/tests/integration/states/git.py b/tests/integration/states/git.py index 7945b1cb2d..ba58fa2288 100644 --- a/tests/integration/states/git.py +++ b/tests/integration/states/git.py @@ -8,6 +8,7 @@ from __future__ import absolute_import import os import shutil import socket +import string import subprocess import tempfile @@ -328,6 +329,136 @@ class GitTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn): self.assertSaltTrueReturn(ret) +@skip_if_binaries_missing('git') +class LocalRepoGitTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn): + ''' + Tests which do no require connectivity to github.com + ''' + def test_renamed_default_branch(self): + ''' + Test the case where the remote branch has been removed + https://github.com/saltstack/salt/issues/36242 + ''' + cwd = os.getcwd() + repo = tempfile.mkdtemp(dir=integration.TMP) + admin = tempfile.mkdtemp(dir=integration.TMP) + name = tempfile.mkdtemp(dir=integration.TMP) + for dirname in (repo, admin, name): + self.addCleanup(shutil.rmtree, dirname, ignore_errors=True) + self.addCleanup(os.chdir, cwd) + + with salt.utils.fopen(os.devnull, 'w') as devnull: + # Create bare repo + subprocess.check_call(['git', 'init', '--bare', repo], + stdout=devnull, stderr=devnull) + # Clone bare repo + subprocess.check_call(['git', 'clone', repo, admin], + stdout=devnull, stderr=devnull) + + # Create, add, commit, and push file + os.chdir(admin) + with salt.utils.fopen('foo', 'w'): + pass + subprocess.check_call(['git', 'add', '.'], + stdout=devnull, stderr=devnull) + subprocess.check_call(['git', 'commit', '-m', 'init'], + stdout=devnull, stderr=devnull) + subprocess.check_call(['git', 'push', 'origin', 'master'], + stdout=devnull, stderr=devnull) + + # Change back to the original cwd + os.chdir(cwd) + + # Rename remote 'master' branch to 'develop' + os.rename( + os.path.join(repo, 'refs', 'heads', 'master'), + os.path.join(repo, 'refs', 'heads', 'develop') + ) + + # Run git.latest state. This should successfuly clone and fail with a + # specific error in the comment field. + ret = self.run_state( + 'git.latest', + name=repo, + target=name, + rev='develop', + ) + self.assertSaltFalseReturn(ret) + self.assertEqual( + ret[next(iter(ret))]['comment'], + 'Remote HEAD refers to a ref that does not exist. ' + 'This can happen when the default branch on the ' + 'remote repository is renamed or deleted. If you ' + 'are unable to fix the remote repository, you can ' + 'work around this by setting the \'branch\' argument ' + '(which will ensure that the named branch is created ' + 'if it does not already exist).\n\n' + 'Changes already made: {0} cloned to {1}' + .format(repo, name) + ) + self.assertEqual( + ret[next(iter(ret))]['changes'], + {'new': '{0} => {1}'.format(repo, name)} + ) + + # Run git.latest state again. This should fail again, with a different + # error in the comment field, and should not change anything. + ret = self.run_state( + 'git.latest', + name=repo, + target=name, + rev='develop', + ) + self.assertSaltFalseReturn(ret) + self.assertEqual( + ret[next(iter(ret))]['comment'], + 'Cannot set/unset upstream tracking branch, local ' + 'HEAD refers to nonexistent branch. This may have ' + 'been caused by cloning a remote repository for which ' + 'the default branch was renamed or deleted. If you ' + 'are unable to fix the remote repository, you can ' + 'work around this by setting the \'branch\' argument ' + '(which will ensure that the named branch is created ' + 'if it does not already exist).' + ) + self.assertEqual(ret[next(iter(ret))]['changes'], {}) + + # Run git.latest state again with a branch manually set. This should + # checkout a new branch and the state should pass. + ret = self.run_state( + 'git.latest', + name=repo, + target=name, + rev='develop', + branch='develop', + ) + # State should succeed + self.assertSaltTrueReturn(ret) + self.assertSaltCommentRegexpMatches( + ret, + 'New branch \'develop\' was checked out, with origin/develop ' + r'\([0-9a-f]{7}\) as a starting point' + ) + # Only the revision should be in the changes dict. + self.assertEqual( + list(ret[next(iter(ret))]['changes'].keys()), + ['revision'] + ) + # Since the remote repo was incorrectly set up, the local head should + # not exist (therefore the old revision should be None). + self.assertEqual( + ret[next(iter(ret))]['changes']['revision']['old'], + None + ) + # Make sure the new revision is a SHA (40 chars, all hex) + self.assertTrue( + len(ret[next(iter(ret))]['changes']['revision']['new']) == 40) + self.assertTrue( + all([x in string.hexdigits for x in + ret[next(iter(ret))]['changes']['revision']['new']]) + ) + + if __name__ == '__main__': from integration import run_tests run_tests(GitTest) From ac5c812e4b1499e15e10b8b1f1a5e71051c27487 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 19 Sep 2016 12:14:43 -0500 Subject: [PATCH 8/9] Fix OS identification for CloudLinux (#36408) --- salt/grains/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/grains/core.py b/salt/grains/core.py index ad874c6892..2c64710a2d 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -925,6 +925,7 @@ _OS_NAME_MAP = { 'enterprise': 'OEL', 'oracleserv': 'OEL', 'cloudserve': 'CloudLinux', + 'cloudlinux': 'CloudLinux', 'pidora': 'Fedora', 'scientific': 'ScientificLinux', 'synology': 'Synology', From 9bc4eeb71eb2199e710bfe12c810f8aaf3b25c8b Mon Sep 17 00:00:00 2001 From: Jacob Hammons Date: Mon, 19 Sep 2016 11:15:03 -0600 Subject: [PATCH 9/9] Fix typo (#36409) --- doc/faq.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/faq.rst b/doc/faq.rst index 8a4b3098b2..a12b514042 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -14,10 +14,10 @@ open and proprietary projects. To expand on this a little: -There is much argument over the actual definition of "open core". From our standpoint, Salt is open source because +There is much argument over the actual definition of "open core". From our standpoint, Salt is open source because -1. It is a standalone product that that anyone is free to use. -2. It is developed in the open with contributions accepted from the community for the good of the project. +1. It is a standalone product that anyone is free to use. +2. It is developed in the open with contributions accepted from the community for the good of the project. 3. There are no features of Salt itself that are restricted to separate proprietary products distributed by SaltStack, Inc. 4. Because of our Apache 2.0 license, Salt can be used as the foundation for a project or even a proprietary tool. 5. Our APIs are open and documented (any lack of documentation is an oversight as opposed to an intentional decision by SaltStack the company) and available for use by anyone.