valitydev/salt
14
0
Fork 0
mirror of https://github.com/valitydev/salt.git synced 2024-11-07 00:55:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch '2016.3' into 'carbon'

Browse Source 
No conflicts.
This commit is contained in:
rallytime 2016-09-22 10:01:17 -06:00
commit 1755f22de1
7 changed files with 376 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
doc/topics/development/contributing.rst
View File

@ -110,26 +110,54 @@ Fork a Repo Guide_>`_ and is well worth reading.
If you get stuck, there are many introductory Git resources on
http://help.github.com.
#. Push your locally-committed changes to your GitHub fork,
#. Push your locally-committed changes to your GitHub fork.
.. code-block:: bash
git push -u origin fix-broken-thing
or
.. code-block:: bash
git push -u origin add-cool-feature
.. note::
You may want to rebase before pushing to work out any potential
conflicts.
conflicts:
.. code-block:: bash
.. code-block:: bash
git fetch upstream
git rebase upstream/2015.5 fix-broken-thing
git push --set-upstream origin fix-broken-thing
git fetch upstream
git rebase upstream/2015.5 fix-broken-thing
git push -u origin fix-broken-thing
or,
or
.. code-block:: bash
.. code-block:: bash
git fetch upstream
git rebase upstream/develop add-cool-feature
git push --set-upstream origin add-cool-feature
git fetch upstream
git rebase upstream/develop add-cool-feature
git push -u origin add-cool-feature
If you do rebase, and the push is rejected with a
``(non-fast-forward)`` comment, then run ``git status``. You will
likely see a message about the branches diverging:
.. code-block:: text
On branch fix-broken-thing
Your branch and 'origin/fix-broken-thing' have diverged,
and have 1 and 2 different commits each, respectively.
(use "git pull" to merge the remote branch into yours)
nothing to commit, working tree clean
Do **NOT** perform a ``git pull`` or ``git merge`` here. Instead, add
``--force`` to the end of the ``git push`` command to get the changes
pushed to your fork. Pulling or merging, while they will resolve the
non-fast-forward issue, will likely add extra commits to the pull
request which were not part of your changes.
#. Find the branch on your GitHub salt fork.

5
doc/topics/tutorials/gitfs.rst
View File

@ -887,7 +887,8 @@ steps to this process:
#!/usr/bin/env sh
salt-call event.fire_master update salt/fileserver/gitfs/update
b. To enable other git users to run the hook after a `push`, use sudo in the hook script:
b. To enable other git users to run the hook after a `push`, use sudo in the hook script:
.. code-block:: bash
#!/usr/bin/env sh
@ -896,7 +897,7 @@ steps to this process:
4. If using sudo in the git hook (above), the policy must be changed to permit all users to fire the event.
Add the following policy to the sudoers file on the git server.
.. code-block::
.. code-block:: bash
Cmnd_Alias SALT_GIT_HOOK = /bin/salt-call event.fire_master update salt/fileserver/gitfs/update
Defaults!SALT_GIT_HOOK !requiretty

9
salt/cloud/clouds/ec2.py
View File

@ -11,13 +11,10 @@ To use the EC2 cloud module, set up the cloud configuration at
.. code-block:: yaml
my-ec2-config:
# The EC2 API authentication id, set this and/or key to
# 'use-instance-role-credentials' to use the instance role credentials
# from the meta-data if running on an AWS instance
# EC2 API credentials: Access Key ID and Secret Access Key.
# Alternatively, to use IAM Instance Role credentials available via
# EC2 metadata set both id and key to 'use-instance-role-credentials'
id: GKTADJGHEIQSXMKKRBJ08H
# The EC2 API authentication key, set this and/or id to
# 'use-instance-role-credentials' to use the instance role credentials
# from the meta-data if running on an AWS instance
key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs
# The ssh keyname to use
keyname: default

55
salt/states/git.py
View File

@ -182,14 +182,39 @@ def _failed_submodule_update(ret, exc, comments=None):
return _fail(ret, msg, comments)
def _not_fast_forward(ret, pre, post, branch, local_branch,
local_changes, comments):
def _not_fast_forward(ret, rev, pre, post, branch, local_branch,
default_branch, local_changes, comments):
branch_msg = ''
if branch is None:
if rev != 'HEAD':
if local_branch != rev:
branch_msg = (
' The desired rev ({0}) differs from the name of the '
'local branch ({1}), if the desired rev is a branch name '
'then a forced update could possibly be avoided by '
'setting the \'branch\' argument to \'{0}\' instead.'
.format(rev, local_branch)
)
else:
if default_branch is not None and local_branch != default_branch:
branch_msg = (
' The default remote branch ({0}) differs from the '
'local branch ({1}). This could be caused by changing the '
'default remote branch, or if the local branch was '
'manually changed. Rather than forcing an update, it '
'may be advisable to set the \'branch\' argument to '
'\'{0}\' instead. To ensure that this state follows the '
'\'{0}\' branch instead of the remote HEAD, set the '
'\'rev\' argument to \'{0}\'.'
.format(default_branch, local_branch)
)
pre = _short_sha(pre)
post = _short_sha(post)
return _fail(
ret,
'Repository would be updated {0}{1}, but {2}. Set \'force_reset\' to '
'True to force this update{3}.'.format(
'True to force this update{3}.{4}'.format(
'from {0} to {1}'.format(pre, post)
if local_changes and pre != post
else 'to {0}'.format(post),
@ -199,7 +224,8 @@ def _not_fast_forward(ret, pre, post, branch, local_branch,
'this is not a fast-forward merge'
if not local_changes
else 'there are uncommitted changes',
' and discard these changes' if local_changes else ''
' and discard these changes' if local_changes else '',
branch_msg,
),
comments
)
@ -614,14 +640,27 @@ def latest(name,
'Failed to check remote refs: {0}'.format(_strip_exc(exc))
)
if 'HEAD' in all_remote_refs:
head_rev = all_remote_refs['HEAD']
for refname, refsha in six.iteritems(all_remote_refs):
if refname.startswith('refs/heads/'):
if refsha == head_rev:
default_branch = refname.partition('refs/heads/')[-1]
break
else:
default_branch = None
else:
head_ref = None
default_branch = None
desired_upstream = False
if bare:
remote_rev = None
remote_rev_type = None
else:
if rev == 'HEAD':
if 'HEAD' in all_remote_refs:
remote_rev = all_remote_refs['HEAD']
if head_rev is not None:
remote_rev = head_rev
# Just go with whatever the upstream currently is
desired_upstream = None
remote_rev_type = 'sha1'
@ -935,10 +974,12 @@ def latest(name,
if not force_reset:
return _not_fast_forward(
ret,
rev,
base_rev,
remote_rev,
branch,
local_branch,
default_branch,
local_changes,
comments)
merge_action = 'hard-reset'
@ -1246,10 +1287,12 @@ def latest(name,
if fast_forward is False and not force_reset:
return _not_fast_forward(
ret,
rev,
base_rev,
remote_rev,
branch,
local_branch,
default_branch,
local_changes,
comments)

369
salt/utils/cloud.py
View File

@ -5,6 +5,7 @@ Utility functions for salt.cloud
# Import python libs
from __future__ import absolute_import
import errno
import os
import sys
import stat
@ -1815,97 +1816,117 @@ def scp_file(dest_path, contents=None, kwargs=None, local_file=None):
'''
Use scp or sftp to copy a file to a server
'''
if contents is not None:
tmpfh, tmppath = tempfile.mkstemp()
with salt.utils.fopen(tmppath, 'w') as tmpfile:
tmpfile.write(contents)
file_to_upload = None
try:
if contents is not None:
try:
tmpfd, file_to_upload = tempfile.mkstemp()
os.write(tmpfd, contents)
finally:
try:
os.close(tmpfd)
except OSError as exc:
if exc.errno != errno.EBADF:
raise exc
log.debug('Uploading {0} to {1}'.format(dest_path, kwargs['hostname']))
log.debug('Uploading {0} to {1}'.format(dest_path, kwargs['hostname']))
ssh_args = [
# Don't add new hosts to the host key database
'-oStrictHostKeyChecking=no',
# Set hosts key database path to /dev/null, i.e., non-existing
'-oUserKnownHostsFile=/dev/null',
# Don't re-use the SSH connection. Less failures.
'-oControlPath=none'
]
ssh_args = [
# Don't add new hosts to the host key database
'-oStrictHostKeyChecking=no',
# Set hosts key database path to /dev/null, i.e., non-existing
'-oUserKnownHostsFile=/dev/null',
# Don't re-use the SSH connection. Less failures.
'-oControlPath=none'
]
if local_file is not None:
tmppath = local_file
if os.path.isdir(local_file):
ssh_args.append('-r')
if local_file is not None:
file_to_upload = local_file
if os.path.isdir(local_file):
ssh_args.append('-r')
if 'key_filename' in kwargs:
# There should never be both a password and an ssh key passed in, so
ssh_args.extend([
# tell SSH to skip password authentication
'-oPasswordAuthentication=no',
'-oChallengeResponseAuthentication=no',
# Make sure public key authentication is enabled
'-oPubkeyAuthentication=yes',
# do only use the provided identity file
'-oIdentitiesOnly=yes',
# No Keyboard interaction!
'-oKbdInteractiveAuthentication=no',
# Also, specify the location of the key file
'-i {0}'.format(kwargs['key_filename'])
])
if 'key_filename' in kwargs:
# There should never be both a password and an ssh key passed in, so
ssh_args.extend([
# tell SSH to skip password authentication
'-oPasswordAuthentication=no',
'-oChallengeResponseAuthentication=no',
# Make sure public key authentication is enabled
'-oPubkeyAuthentication=yes',
# do only use the provided identity file
'-oIdentitiesOnly=yes',
# No Keyboard interaction!
'-oKbdInteractiveAuthentication=no',
# Also, specify the location of the key file
'-i {0}'.format(kwargs['key_filename'])
])
if 'port' in kwargs:
ssh_args.append('-oPort={0}'.format(kwargs['port']))
if 'port' in kwargs:
ssh_args.append('-oPort={0}'.format(kwargs['port']))
if 'ssh_gateway' in kwargs:
ssh_gateway = kwargs['ssh_gateway']
ssh_gateway_port = 22
ssh_gateway_key = ''
ssh_gateway_user = 'root'
if ':' in ssh_gateway:
ssh_gateway, ssh_gateway_port = ssh_gateway.split(':')
if 'ssh_gateway_port' in kwargs:
ssh_gateway_port = kwargs['ssh_gateway_port']
if 'ssh_gateway_key' in kwargs:
ssh_gateway_key = '-i {0}'.format(kwargs['ssh_gateway_key'])
if 'ssh_gateway_user' in kwargs:
ssh_gateway_user = kwargs['ssh_gateway_user']
if 'ssh_gateway' in kwargs:
ssh_gateway = kwargs['ssh_gateway']
ssh_gateway_port = 22
ssh_gateway_key = ''
ssh_gateway_user = 'root'
if ':' in ssh_gateway:
ssh_gateway, ssh_gateway_port = ssh_gateway.split(':')
if 'ssh_gateway_port' in kwargs:
ssh_gateway_port = kwargs['ssh_gateway_port']
if 'ssh_gateway_key' in kwargs:
ssh_gateway_key = '-i {0}'.format(kwargs['ssh_gateway_key'])
if 'ssh_gateway_user' in kwargs:
ssh_gateway_user = kwargs['ssh_gateway_user']
ssh_args.append(
# Setup ProxyCommand
'-oProxyCommand="ssh {0} {1} {2} {3} {4}@{5} -p {6} nc -q0 %h %p"'.format(
# Don't add new hosts to the host key database
'-oStrictHostKeyChecking=no',
# Set hosts key database path to /dev/null, i.e., non-existing
'-oUserKnownHostsFile=/dev/null',
# Don't re-use the SSH connection. Less failures.
'-oControlPath=none',
ssh_gateway_key,
ssh_gateway_user,
ssh_gateway,
ssh_gateway_port
ssh_args.append(
# Setup ProxyCommand
'-oProxyCommand="ssh {0} {1} {2} {3} {4}@{5} -p {6} nc -q0 %h %p"'.format(
# Don't add new hosts to the host key database
'-oStrictHostKeyChecking=no',
# Set hosts key database path to /dev/null, i.e., non-existing
'-oUserKnownHostsFile=/dev/null',
# Don't re-use the SSH connection. Less failures.
'-oControlPath=none',
ssh_gateway_key,
ssh_gateway_user,
ssh_gateway,
ssh_gateway_port
)
)
try:
if socket.inet_pton(socket.AF_INET6, kwargs['hostname']):
ipaddr = '[{0}]'.format(kwargs['hostname'])
else:
ipaddr = kwargs['hostname']
except socket.error:
ipaddr = kwargs['hostname']
if file_to_upload is None:
log.warning(
'No source file to upload. Please make sure that either file '
'contents or the path to a local file are provided.'
)
cmd = (
'scp {0} {1} {2[username]}@{4}:{3} || '
'echo "put {1} {3}" | sftp {0} {2[username]}@{4} || '
'rsync -avz -e "ssh {0}" {1} {2[username]}@{2[hostname]}:{3}'.format(
' '.join(ssh_args), file_to_upload, kwargs, dest_path, ipaddr
)
)
try:
if socket.inet_pton(socket.AF_INET6, kwargs['hostname']):
ipaddr = '[{0}]'.format(kwargs['hostname'])
else:
ipaddr = kwargs['hostname']
except socket.error:
ipaddr = kwargs['hostname']
cmd = (
'scp {0} {1} {2[username]}@{4}:{3} || '
'echo "put {1} {3}" | sftp {0} {2[username]}@{4} || '
'rsync -avz -e "ssh {0}" {1} {2[username]}@{2[hostname]}:{3}'.format(
' '.join(ssh_args), tmppath, kwargs, dest_path, ipaddr
)
)
log.debug('SCP command: \'{0}\''.format(cmd))
retcode = _exec_ssh_cmd(cmd,
error_msg='Failed to upload file \'{0}\': {1}\n{2}',
password_retries=3,
**kwargs)
log.debug('SCP command: \'{0}\''.format(cmd))
retcode = _exec_ssh_cmd(cmd,
error_msg='Failed to upload file \'{0}\': {1}\n{2}',
password_retries=3,
**kwargs)
finally:
if contents is not None:
try:
os.remove(file_to_upload)
except OSError as exc:
if exc.errno != errno.ENOENT:
raise exc
return retcode
@ -1928,91 +1949,111 @@ def sftp_file(dest_path, contents=None, kwargs=None, local_file=None):
if kwargs is None:
kwargs = {}
if contents is not None:
tmpfh, tmppath = tempfile.mkstemp()
with salt.utils.fopen(tmppath, 'w') as tmpfile:
tmpfile.write(contents)
if local_file is not None:
tmppath = local_file
if os.path.isdir(local_file):
put_args = ['-r']
log.debug('Uploading {0} to {1} (sftp)'.format(dest_path, kwargs.get('hostname')))
ssh_args = [
# Don't add new hosts to the host key database
'-oStrictHostKeyChecking=no',
# Set hosts key database path to /dev/null, i.e., non-existing
'-oUserKnownHostsFile=/dev/null',
# Don't re-use the SSH connection. Less failures.
'-oControlPath=none'
]
if 'key_filename' in kwargs:
# There should never be both a password and an ssh key passed in, so
ssh_args.extend([
# tell SSH to skip password authentication
'-oPasswordAuthentication=no',
'-oChallengeResponseAuthentication=no',
# Make sure public key authentication is enabled
'-oPubkeyAuthentication=yes',
# do only use the provided identity file
'-oIdentitiesOnly=yes',
# No Keyboard interaction!
'-oKbdInteractiveAuthentication=no',
# Also, specify the location of the key file
'-oIdentityFile={0}'.format(kwargs['key_filename'])
])
if 'port' in kwargs:
ssh_args.append('-oPort={0}'.format(kwargs['port']))
if 'ssh_gateway' in kwargs:
ssh_gateway = kwargs['ssh_gateway']
ssh_gateway_port = 22
ssh_gateway_key = ''
ssh_gateway_user = 'root'
if ':' in ssh_gateway:
ssh_gateway, ssh_gateway_port = ssh_gateway.split(':')
if 'ssh_gateway_port' in kwargs:
ssh_gateway_port = kwargs['ssh_gateway_port']
if 'ssh_gateway_key' in kwargs:
ssh_gateway_key = '-i {0}'.format(kwargs['ssh_gateway_key'])
if 'ssh_gateway_user' in kwargs:
ssh_gateway_user = kwargs['ssh_gateway_user']
ssh_args.append(
# Setup ProxyCommand
'-oProxyCommand="ssh {0} {1} {2} {3} {4}@{5} -p {6} nc -q0 %h %p"'.format(
# Don't add new hosts to the host key database
'-oStrictHostKeyChecking=no',
# Set hosts key database path to /dev/null, i.e., non-existing
'-oUserKnownHostsFile=/dev/null',
# Don't re-use the SSH connection. Less failures.
'-oControlPath=none',
ssh_gateway_key,
ssh_gateway_user,
ssh_gateway,
ssh_gateway_port
)
)
file_to_upload = None
try:
if socket.inet_pton(socket.AF_INET6, kwargs['hostname']):
ipaddr = '[{0}]'.format(kwargs['hostname'])
else:
ipaddr = kwargs['hostname']
except socket.error:
ipaddr = kwargs['hostname']
if contents is not None:
try:
tmpfd, file_to_upload = tempfile.mkstemp()
os.write(tmpfd, contents)
finally:
try:
os.close(tmpfd)
except OSError as exc:
if exc.errno != errno.EBADF:
raise exc
cmd = 'echo "put {0} {1} {2}" | sftp {3} {4[username]}@{5}'.format(
' '.join(put_args), tmppath, dest_path, ' '.join(ssh_args), kwargs, ipaddr
)
log.debug('SFTP command: \'{0}\''.format(cmd))
retcode = _exec_ssh_cmd(cmd,
error_msg='Failed to upload file \'{0}\': {1}\n{2}',
password_retries=3,
**kwargs)
if local_file is not None:
file_to_upload = local_file
if os.path.isdir(local_file):
put_args = ['-r']
log.debug('Uploading {0} to {1} (sftp)'.format(dest_path, kwargs.get('hostname')))
ssh_args = [
# Don't add new hosts to the host key database
'-oStrictHostKeyChecking=no',
# Set hosts key database path to /dev/null, i.e., non-existing
'-oUserKnownHostsFile=/dev/null',
# Don't re-use the SSH connection. Less failures.
'-oControlPath=none'
]
if 'key_filename' in kwargs:
# There should never be both a password and an ssh key passed in, so
ssh_args.extend([
# tell SSH to skip password authentication
'-oPasswordAuthentication=no',
'-oChallengeResponseAuthentication=no',
# Make sure public key authentication is enabled
'-oPubkeyAuthentication=yes',
# do only use the provided identity file
'-oIdentitiesOnly=yes',
# No Keyboard interaction!
'-oKbdInteractiveAuthentication=no',
# Also, specify the location of the key file
'-oIdentityFile={0}'.format(kwargs['key_filename'])
])
if 'port' in kwargs:
ssh_args.append('-oPort={0}'.format(kwargs['port']))
if 'ssh_gateway' in kwargs:
ssh_gateway = kwargs['ssh_gateway']
ssh_gateway_port = 22
ssh_gateway_key = ''
ssh_gateway_user = 'root'
if ':' in ssh_gateway:
ssh_gateway, ssh_gateway_port = ssh_gateway.split(':')
if 'ssh_gateway_port' in kwargs:
ssh_gateway_port = kwargs['ssh_gateway_port']
if 'ssh_gateway_key' in kwargs:
ssh_gateway_key = '-i {0}'.format(kwargs['ssh_gateway_key'])
if 'ssh_gateway_user' in kwargs:
ssh_gateway_user = kwargs['ssh_gateway_user']
ssh_args.append(
# Setup ProxyCommand
'-oProxyCommand="ssh {0} {1} {2} {3} {4}@{5} -p {6} nc -q0 %h %p"'.format(
# Don't add new hosts to the host key database
'-oStrictHostKeyChecking=no',
# Set hosts key database path to /dev/null, i.e., non-existing
'-oUserKnownHostsFile=/dev/null',
# Don't re-use the SSH connection. Less failures.
'-oControlPath=none',
ssh_gateway_key,
ssh_gateway_user,
ssh_gateway,
ssh_gateway_port
)
)
try:
if socket.inet_pton(socket.AF_INET6, kwargs['hostname']):
ipaddr = '[{0}]'.format(kwargs['hostname'])
else:
ipaddr = kwargs['hostname']
except socket.error:
ipaddr = kwargs['hostname']
if file_to_upload is None:
log.warning(
'No source file to upload. Please make sure that either file '
'contents or the path to a local file are provided.'
)
cmd = 'echo "put {0} {1} {2}" | sftp {3} {4[username]}@{5}'.format(
' '.join(put_args), file_to_upload, dest_path, ' '.join(ssh_args), kwargs, ipaddr
)
log.debug('SFTP command: \'{0}\''.format(cmd))
retcode = _exec_ssh_cmd(cmd,
error_msg='Failed to upload file \'{0}\': {1}\n{2}',
password_retries=3,
**kwargs)
finally:
if contents is not None:
try:
os.remove(file_to_upload)
except OSError as exc:
if exc.errno != errno.ENOENT:
raise exc
return retcode

3
salt/utils/context.py
View File

@ -181,6 +181,9 @@ class NamespacedDictWrapper(collections.MutableMapping, dict):
r = r[k]
return r
def __repr__(self):
return repr(self._dict())
def __setitem__(self, key, val):
self._dict()[key] = val

74
tests/integration/states/git.py
View File

@ -260,6 +260,80 @@ class GitTest(integration.ModuleCase, integration.SaltReturnAssertsMixIn):
for path in (mirror_dir, admin_dir, clone_dir):
shutil.rmtree(path, ignore_errors=True)
def _changed_local_branch_helper(self, rev, hint):
'''
We're testing two almost identical cases, the only thing that differs
is the rev used for the git.latest state.
'''
name = os.path.join(integration.TMP, 'salt_repo')
cwd = os.getcwd()
try:
# Clone repo
ret = self.run_state(
'git.latest',
name='https://{0}/saltstack/salt-test-repo.git'.format(self.__domain),
rev=rev,
target=name
)
self.assertSaltTrueReturn(ret)
# Check out a new branch in the clone and make a commit, to ensure
# that when we re-run the state, it is not a fast-forward change
os.chdir(name)
with salt.utils.fopen(os.devnull, 'w') as devnull:
subprocess.check_call(['git', 'checkout', '-b', 'new_branch'],
stdout=devnull, stderr=devnull)
with salt.utils.fopen('foo', 'w'):
pass
subprocess.check_call(['git', 'add', '.'],
stdout=devnull, stderr=devnull)
subprocess.check_call(['git', 'commit', '-m', 'add file'],
stdout=devnull, stderr=devnull)
os.chdir(cwd)
# Re-run the state, this should fail with a specific hint in the
# comment field.
ret = self.run_state(
'git.latest',
name='https://{0}/saltstack/salt-test-repo.git'.format(self.__domain),
rev=rev,
target=name
)
self.assertSaltFalseReturn(ret)
comment = ret[next(iter(ret))]['comment']
self.assertTrue(hint in comment)
finally:
# Make sure that we change back to the original cwd even if there
# was a traceback in the test.
os.chdir(cwd)
shutil.rmtree(name, ignore_errors=True)
def test_latest_changed_local_branch_rev_head(self):
'''
Test for presence of hint in failure message when the local branch has
been changed and a the rev is set to HEAD
This test will fail if the default branch for the salt-test-repo is
ever changed.
'''
self._changed_local_branch_helper(
'HEAD',
'The default remote branch (develop) differs from the local '
'branch (new_branch)'
)
def test_latest_changed_local_branch_rev_develop(self):
'''
Test for presence of hint in failure message when the local branch has
been changed and a non-HEAD rev is specified
'''
self._changed_local_branch_helper(
'develop',
'The desired rev (develop) differs from the name of the local '
'branch (new_branch)'
)
def test_present(self):
'''
git.present