From 83dc9a34f654869194dd661d730460d1298b9eaf Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 19 Aug 2015 15:21:28 -0500 Subject: [PATCH 01/13] Tweak top-level RST in git state module --- salt/states/git.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/salt/states/git.py b/salt/states/git.py index 0810aa0e63..4294883a4c 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -1,18 +1,10 @@ # -*- coding: utf-8 -*- ''' -Interaction with Git repositories -================================= +States to manage git repositories and git configuration -Important: Before using git over ssh, make sure your remote host fingerprint -exists in your ``~/.ssh/known_hosts`` file. To avoid requiring password -authentication, it is also possible to pass private keys to use explicitly. - -.. code-block:: yaml - - https://github.com/saltstack/salt.git: - git.latest: - - rev: develop - - target: /tmp/salt +.. important:: + Before using git over ssh, make sure your remote host fingerprint exists in + your ``~/.ssh/known_hosts`` file. ''' from __future__ import absolute_import From 1be2a8a1a1d936353f82f697a5553d8b0e14e3e2 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 19 Aug 2015 15:22:04 -0500 Subject: [PATCH 02/13] Add config examples to git_pillar top-level docstring Also, hide the legacy git_pillar code from sphinx by prefixing the class name with an underscore. --- salt/pillar/git_pillar.py | 53 ++++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/salt/pillar/git_pillar.py b/salt/pillar/git_pillar.py index d768c7847a..f7a14a0e01 100644 --- a/salt/pillar/git_pillar.py +++ b/salt/pillar/git_pillar.py @@ -26,6 +26,8 @@ The branch/tag which maps to that environment must then be specified along with the repo's URL. Configuration details can be found below. +.. _git-pillar-pre-2015-8-0: + Configuring git_pillar for Salt releases before 2015.8.0 ======================================================== @@ -96,13 +98,22 @@ The corresponding Pillar top file would look like this: '*': - bar +.. _git-pillar-2015-8-0-and-later: + Configuring git_pillar for Salt releases 2015.8.0 and later =========================================================== -Beginning with Salt version 2015.8.0, pygit2_ is now supported for git_pillar, -in addition to GitPython_ (Dulwich_ will not be supported for the forseeable -future). The requirements for GitPython_ and pygit2_ are the same as for gitfs, -as described :ref:`here `. +.. note:: + In version 2015.8.0, the method of configuring git external pillars has + changed, and now more closely resembles that of the :ref:`Git Fileserver + Backend `. If Salt detects the old configuration schema, it + will use the pre-2015.8.0 code to compile the external pillar. A warning + will also be logged. + +Beginning with Salt version 2015.8.0, pygit2_ is now supported in addition to +GitPython_ (Dulwich_ will not be supported for the forseeable future). The +requirements for GitPython_ and pygit2_ are the same as for gitfs, as described +:ref:`here `. Here is an example git_pillar configuration. @@ -110,13 +121,27 @@ Here is an example git_pillar configuration. ext_pillar: - git: + # Use 'prod' instead of the branch name 'production' as the environment - production https://gitserver/git-pillar.git: - env: prod + # Use 'dev' instead of the branch name 'develop' as the environment - develop https://gitserver/git-pillar.git: - env: dev + # No per-remote config parameters (and no trailing colon), 'qa' will + # be used as the environment - qa https://gitserver/git-pillar.git - - master https://other-git-server/pillardata.git + # SSH key authentication + - master git@other-git-server:pillardata-ssh.git: + # Pillar SLS files will be read from the 'pillar' subdirectory in + # this repository - root: pillar + - privkey: /path/to/key + - pubkey: /path/to/key.pub + - passphrase: CorrectHorseBatteryStaple + # HTTPS authentication + - master https://other-git-server/pillardata-https.git: + - user: git + - password: CorrectHorseBatteryStaple The main difference between this and the old way of configuring git_pillar is that multiple remotes can be configured under one ``git`` section under @@ -129,10 +154,10 @@ configuration parameters can also be set. With the addition of pygit2_ support, git_pillar can now interact with authenticated remotes. Authentication works just like in gitfs (as outlined in -the :ref:`GitFS Walkthrough `), only with the global -authenication parameter names prefixed with ``git_pillar`` instead of ``gitfs`` -(e.g. :conf_master:`git_pillar_pubkey`, :conf_master:`git_pillar_privkey`, -:conf_master:`git_pillar_passphrase`, etc.). +the :ref:`Git Fileserver Backend Walkthrough `), only +with the global authenication parameter names prefixed with ``git_pillar`` +instead of ``gitfs`` (e.g. :conf_master:`git_pillar_pubkey`, +:conf_master:`git_pillar_privkey`, :conf_master:`git_pillar_passphrase`, etc.). A full list of the git_pillar configuration options can be found :ref:`here `. @@ -209,7 +234,7 @@ def __virtual__(): def ext_pillar(minion_id, repo, pillar_dirs): ''' - Execute a command and read the output as YAML + Checkout the ext_pillar sources and compile the resulting pillar SLS ''' if isinstance(repo, six.string_types): return _legacy_git_pillar(minion_id, repo, pillar_dirs) @@ -228,7 +253,7 @@ def ext_pillar(minion_id, repo, pillar_dirs): # Legacy git_pillar code -class LegacyGitPillar(object): +class _LegacyGitPillar(object): ''' Deal with the remote git repository for Pillar ''' @@ -363,7 +388,7 @@ def _legacy_git_pillar(minion_id, repo_string, pillar_dirs): # environment is "different" from the branch branch, _, environment = branch_env.partition(':') - gitpil = LegacyGitPillar(branch, repo_location, __opts__) + gitpil = _LegacyGitPillar(branch, repo_location, __opts__) branch = gitpil.branch if environment == '': @@ -402,7 +427,7 @@ def _update(branch, repo_location): return boolean whether it worked ''' - gitpil = LegacyGitPillar(branch, repo_location, __opts__) + gitpil = _LegacyGitPillar(branch, repo_location, __opts__) return gitpil.update() @@ -411,7 +436,7 @@ def _envs(branch, repo_location): ''' Return a list of refs that can be used as environments ''' - gitpil = LegacyGitPillar(branch, repo_location, __opts__) + gitpil = _LegacyGitPillar(branch, repo_location, __opts__) return gitpil.envs() From 52156df14447840481c09ca556f9ae9f56bcac81 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 19 Aug 2015 15:23:25 -0500 Subject: [PATCH 03/13] Docstring fixes in git execution module --- salt/modules/git.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index 8f75e729ea..45646ec866 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -779,6 +779,8 @@ def config_get(key, If ``True``, query the global git configuraton. Otherwise, only the local git configuration will be queried. + .. versionadded:: 2015.8.0 + all : False If ``True``, return a list of all values set for ``key``. If the key does not exist, ``None`` will be returned. @@ -861,9 +863,6 @@ def config_get_regexp(key, cwd The path to the git checkout - .. versionchanged:: 2015.8.0 - Now optional if ``global`` is set to ``True`` - global : False If ``True``, query the global git configuraton. Otherwise, only the local git configuration will be queried. @@ -928,11 +927,6 @@ def config_set(key, The path to the git checkout. Must be an absolute path, or the word ``global`` to indicate that a global key should be set. - .. versionchanged:: 2015.8.0 - Can now be set to ``global`` instead of an absolute path, to set a - global git configuration parameter, deprecating the ``is_global`` - parameter. - .. versionchanged:: 2014.7.0 Made ``cwd`` argument optional if ``is_global=True`` @@ -950,7 +944,7 @@ def config_set(key, Argument renamed from ``setting_value`` to ``value`` add : False - Add a value to a multivar + Add a value to a key, creating/updating a multivar .. versionadded:: 2015.8.0 @@ -1675,11 +1669,6 @@ def merge(cwd, The remote branch or revision to merge into the current branch Revision to merge into the current branch - .. versionchanged:: 2015.8.0 - Default value changed from ``'@{upstream}'`` to ``None`` (unset), - allowing this function to merge the remote tracking branch without - having to specify it - .. deprecated:: 2015.8.0 Use ``rev`` instead. @@ -2035,8 +2024,6 @@ def push(cwd, ignore_retcode=False, branch=None): ''' - .. versionchanged:: 2015.8.0 - Interface to `git-push(1)`_ cwd @@ -2613,7 +2600,7 @@ def stash(cwd, action='save', opts='', user=None, ignore_retcode=False): def status(cwd, user=None, ignore_retcode=False): ''' .. versionchanged:: 2015.8.0 - Return data has changed from a list of tuples to a dictionary + Return data has changed from a list of lists to a dictionary Returns the changes to the repository @@ -2833,6 +2820,8 @@ def symbolic_ref(cwd, def version(versioninfo=False, user=None): ''' + .. versionadded:: 2015.8.0 + Returns the version of Git installed on the minion versioninfo : False From ee1eac925af1bb2befa154c342650c5a435ff2c6 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 19 Aug 2015 15:23:52 -0500 Subject: [PATCH 04/13] Point git_pillar section in gitfs tutorial at git_pillar docstring --- doc/topics/tutorials/gitfs.rst | 43 ++++++++-------------------------- 1 file changed, 10 insertions(+), 33 deletions(-) diff --git a/doc/topics/tutorials/gitfs.rst b/doc/topics/tutorials/gitfs.rst index 004dcb6794..9caaa5cc9e 100644 --- a/doc/topics/tutorials/gitfs.rst +++ b/doc/topics/tutorials/gitfs.rst @@ -739,45 +739,22 @@ anything, so long as the usage is consistent. .. _`post-receive hook`: http://www.git-scm.com/book/en/Customizing-Git-Git-Hooks#Server-Side-Hooks +.. _git-as-ext_pillar Using Git as an External Pillar Source ====================================== -Git repositories can also be used to provide :doc:`Pillar -` data, using the :doc:`External Pillar -` system. Note that this is different -from gitfs, and is not yet at feature parity with it. +The git external pillar (a.k.a. git_pillar) has been rewritten for the 2015.8.0 +release. This rewrite brings with it pygit2_ support (allowing for access to +authenticated repositories), as well as more granular support for per-remote +configuration. -To define a git external pillar, add a section like the following to the salt -master config file: +To make use of the new features, changes to the git ext_pillar configuration +must be made. The new configuration schema is detailed :ref:`here +`. -.. code-block:: yaml - - ext_pillar: - - git: [root=] - -.. versionchanged:: 2014.7.0 - The optional ``root`` parameter was added - -The ```` param is the branch containing the pillar SLS tree. The -```` param is the URI for the repository. To add the -``master`` branch of the specified repo as an external pillar source: - -.. code-block:: yaml - - ext_pillar: - - git: master https://domain.com/pillar.git - -Use the ``root`` parameter to use pillars from a subdirectory of a git -repository: - -.. code-block:: yaml - - ext_pillar: - - git: master https://domain.com/pillar.git root=subdirectory - -More information on the git external pillar can be found in the -:mod:`salt.pillar.git_pillar docs `. +For Salt releases before 2015.8.0, click :ref:`here ` +for documentation. .. _faq-gitfs-bug: From dbeffe494400cc920f2091d8593a0ce07849e099 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 19 Aug 2015 15:24:30 -0500 Subject: [PATCH 05/13] Point git_pillar reference in winrepo documentation at a valid ref --- doc/topics/windows/windows-package-manager.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/topics/windows/windows-package-manager.rst b/doc/topics/windows/windows-package-manager.rst index 8e86bc4bdf..0d35928f92 100644 --- a/doc/topics/windows/windows-package-manager.rst +++ b/doc/topics/windows/windows-package-manager.rst @@ -342,10 +342,10 @@ list of the winrepo config options, see :ref:`here `. Starting in version 2015.8.0, the :py:func:`winrepo.update_git_repos ` runner now makes use of the same underlying code used by the :ref:`Git Fileserver Backend ` and -:ref:`Git External Pillar ` to maintain and update its -local clones of git repositories. If a compatible version of either pygit2_ -(0.20.3 and later) or GitPython_ (0.3.0 or later) is installed, then Salt will -use it instead of the old method (which invokes the :py:func:`git.latest +:ref:`Git External Pillar ` to maintain and update its local +clones of git repositories. If a compatible version of either pygit2_ (0.20.3 +and later) or GitPython_ (0.3.0 or later) is installed, then Salt will use it +instead of the old method (which invokes the :py:func:`git.latest ` state). .. note:: @@ -388,7 +388,7 @@ example of this would be the following: - passphrase: myaw3s0m3pa$$phr4$3 - https://github.com/myuser/privaterepo.git: - user: mygithubuser - - password: correcthorsebatterystaple + - password: CorrectHorseBatteryStaple .. note:: Per-remote configuration settings work in the same fashion as they do in From a4b74576836225027a8c396b72a19e6bda5bbb39 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 19 Aug 2015 15:25:07 -0500 Subject: [PATCH 06/13] Add information re: git state/module rewrite Also improve the git_pillar information --- doc/topics/releases/beryllium.rst | 177 +++++++++++++++++++++++++++++- 1 file changed, 173 insertions(+), 4 deletions(-) diff --git a/doc/topics/releases/beryllium.rst b/doc/topics/releases/beryllium.rst index 1c73598768..84b9db5657 100644 --- a/doc/topics/releases/beryllium.rst +++ b/doc/topics/releases/beryllium.rst @@ -31,13 +31,182 @@ A new docker :mod:`state ` and :mod:`execution module of the existing state and execution module, but for now will exist alongside them. +Git State and Execution Modules Rewritten +========================================= + +The git state and execution modules have gone through an extensive overhaul. + +Changes in the :py:func:`git.latest ` State +------------------------------------------------------------------- + +- The ``branch`` parameter has been added, allowing for a custom branch name to + be used in the local checkout maintained by the :py:func:`git.latest + ` state. This can be helpful in avoiding ambiguous + refs in the local checkout when a tag is used as the ``rev`` parameter. If no + ``branch`` is specified, then the state uses the value of ``rev`` as the + branch name. +- The ``remote_name`` parameter has been deprecated and renamed to ``remote``. +- The ``force`` parameter has been deprecated and renamed to ``force_clone`` to + reduce ambiguity with the other "force" parameters. +- Using SHA1 hashes (full or shortened) in the ``rev`` parameter is now + properly supported. +- Non-fast-forward merges are now detected before the repository is updated, + and the state will not update the repository if the change is not a + fast-forward. Non-fast-forward updates must be overridden with the + ``force_reset`` parameter. If ``force_reset`` is set to ``True``, the state + will only reset the repository if it cannot be fast-forwarded. This is in + contrast to the earlier behavior, in which a hard-reset would be performed + every time the state was run if ``force_reset`` was set to ``True``. +- A ``git pull`` is no longer performed by this state, dropped in favor of a + fetch-and-merge (or fetch-and-reset) workflow. + +:py:func:`git.config_unset ` state added +---------------------------------------------------------------------- + +This state allows for configuration values (or entire keys) to be unset. See +:py:func:`here ` for more information and example +SLS. + +git.config State Renamed to :py:func:`git.config_set ` +---------------------------------------------------------------------------------- + +To reduce confusion after the addition of :py:func:`git.config_unset +`, the git.config state has been renamed to +:py:func:`git.config_set `. The old config.get name +will still work for a couple releases, allowing time for SLS files to be +updated. + +In addition, this state now supports managing multivar git configuration +values. See :py:func:`here ` for more information +and example SLS. + +Initial Support for Git Worktrees in Execution Module +----------------------------------------------------- + +Several functions have been added to the execution module to manage worktrees_ +(a feature new to Git 2.5.0). State support does not exist yet, but will follow +soon. + +.. _worktrees: http://git-scm.com/docs/git-worktree + +New Functions in Git Execution Module +------------------------------------- + +- :py:func:`git.config_get_regexp ` +- :py:func:`git.config_unset ` +- :py:func:`git.is_worktree ` +- :py:func:`git.list_branches ` +- :py:func:`git.list_tags ` +- :py:func:`git.list_worktrees ` +- :py:func:`git.merge_base ` +- :py:func:`git.merge_tree ` +- :py:func:`git.rev_parse ` +- :py:func:`git.version ` +- :py:func:`git.worktree_rm ` +- :py:func:`git.worktree_add ` +- :py:func:`git.worktree_prune ` + +Changes to Functions in Git Execution Module +-------------------------------------------- + +:py:func:`git.add ` +**************************************** + +- ``--verbose`` is now implied when running the ``git add`` command, to provide + a list of the files added in the return data. + +:py:func:`git.archive ` +************************************************* + +- Now returns ``True`` when the ``git archive`` command was successful, and + otherwise raises an error. +- ``overwrite`` argument added to prevent an exixting archive from being + overwritten by this function. +- ``fmt`` argument deprecated and renamed to ``format`` +- Trailing slash no longer implied in ``prefix`` argument, must be included if + this argument is passed. + +:py:func:`git.checkout ` +*************************************************** + +- The ``rev`` argument is now optional when using ``-b`` or ``-B`` in ``opts``, + allowing for a branch to be created (or reset) using ``HEAD`` as the starting + point. + +:py:func:`git.clone ` +********************************************* + +- The ``name`` argument has been added to specify the name of the directory in + which to clone the repository. If this option is specified, then the clone + will be made within the directory specified by the ``cwd``, instead of at + that location. +- ``repository`` argument deprecated and renamed to ``url`` + +:py:func:`git.config_get ` +******************************************************* + +- ``setting_name`` argument deprecated and renamed to ``key`` +- The ``global`` argument has been added, to query the global git configuration +- The ``all`` argument has been added to return a list of all values for the + specified key, allowing for all values in a multivar to be returned. +- ``cwd`` argument is now optional if ``global`` is set to ``True`` + +:py:func:`git.config_set ` +******************************************************* + +- The value(s) of the key being set are now returned +- ``setting_name`` argument deprecated and renamed to ``key`` +- ``setting_value`` argument deprecated and renamed to ``value`` +- ``is_global`` argument deprecated and renamed to ``global`` +- The ``multivar`` argument has been added to specify a list of values to set + for the specified key. The ``value`` argument is not compatible with + ``multivar``. +- The ``add`` argument has been added to add a value to a key (this essentially + just adds an ``--add`` to the ``git config`` command that is run to set the + value). + +:py:func:`git.ls_remote ` +***************************************************** + +- ``repository`` argument deprecated and renamed to ``remote`` +- ``branch`` argument deprecated and renamed to ``ref`` +- The ``opts`` argument has been added to allow for additional CLI options to + be passed to the ``git ls-remote`` command. + +:py:func:`git.merge ` +********************************************* + +- The ``branch`` argument deprecated and renamed to ``rev`` + +:py:func:`git.status ` +*********************************************** + +- Return data has been changed from a list of lists to a dictionary containing + lists of files in the modified, added, deleted, and untracked states. + +:py:func:`git.submodule ` +***************************************************** + +- Added the ``command`` argument to allow for operations other than ``update`` + to be run on submodules, and deprecated the ``init`` argument. To do a + submodule update with ``init=True`` moving forward, use ``command=update + opts='--init'`` + + Git Pillar Rewritten ==================== -The Git external pillar has been rewritten to bring it up to feature parity -with :mod:`gitfs `. See :mod:`here -` for more information on the new git_pillar -functionality. +The git external pillar has been rewritten to bring it up to feature parity +with :mod:`gitfs `. Support for pygit2_ has been added, +bring with it the ability to access authenticated repositories. + +Using the new features will require updates to the git ext_pillar +configuration, further details can be found :ref:`here +`. + +.. note:: + As with :mod:`gitfs `, pygit2_ 0.20.3 is required to + use pygit2_ with the git external pillar. Windows Software Repo Changes ============================= From 57f0cfa6486016539117c07073e31865e8ecd29d Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 19 Aug 2015 19:05:08 -0500 Subject: [PATCH 07/13] Fix reference to old LegacyGitPillar class --- salt/daemons/masterapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/daemons/masterapi.py b/salt/daemons/masterapi.py index a1da41db06..3938385da6 100644 --- a/salt/daemons/masterapi.py +++ b/salt/daemons/masterapi.py @@ -78,7 +78,7 @@ def init_git_pillar(opts): ) else: ret.append( - git_pillar.LegacyGitPillar( + git_pillar._LegacyGitPillar( br, loc, opts From b4773e8d07aa4df9c382feff34f37899db708842 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 19 Aug 2015 19:05:45 -0500 Subject: [PATCH 08/13] Use raw string for regex --- salt/states/git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/git.py b/salt/states/git.py index 4294883a4c..9fa5268c6e 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -101,7 +101,7 @@ def _strip_exc(exc): Strip the actual command that was run from exc.strerror to leave just the error message ''' - return re.sub('^Command [\'"].+[\'"] failed: ', '', exc.strerror) + return re.sub(r'^Command [\'"].+[\'"] failed: ', '', exc.strerror) def _neutral_test(ret, comment): From 4c87aedb9c6b3e76cff527e90c62b6d12469266e Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 19 Aug 2015 19:06:42 -0500 Subject: [PATCH 09/13] 2 fixes, one new function in git execution module 1. Allow rev=None in git.rev_parse 2. When an error is raised by _git_run, join the "command" list to make the error message easier to read Added list_worktrees function. --- salt/modules/git.py | 118 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 113 insertions(+), 5 deletions(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index 45646ec866..6c4a2b48dd 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -6,6 +6,7 @@ from __future__ import absolute_import # Import python libs import copy +import errno import logging import os import shlex @@ -231,7 +232,10 @@ def _git_run(command, cwd=None, runas=None, identity=None, return result else: if failhard: - msg = 'Command \'{0}\' failed'.format(command) + gitcommand = ' '.join(command) \ + if isinstance(command, list) \ + else command + msg = 'Command \'{0}\' failed'.format(gitcommand) if result['stderr']: msg += ': {0}'.format(result['stderr']) raise CommandExecutionError(msg) @@ -1542,6 +1546,103 @@ def list_tags(cwd, user=None, ignore_retcode=False): ignore_retcode=ignore_retcode)['stdout'].splitlines() +def list_worktrees(cwd, stale=False, user=None, **kwargs): + ''' + .. versionadded:: 2015.8.0 + + Return a dictionary mapping worktrees to their locations. + + .. note:: + This information is compiled by analyzing the administrative data in + $GIT_DIR/worktrees. By default, only worktrees for which the gitdir is + still present are returned, but this can be changed using the ``all`` + and ``stale`` arguments (described below). + + cwd + The path to the git checkout + + user + User under which to run the git command. By default, the command is run + by the user under which the minion is running. + + all : False + If ``True``, then return all worktrees, including ones whose gitdir is + no longer present. + + stale : False + If ``True``, return only worktrees whose gitdir is no longer present. + + + CLI Examples: + + .. code-block:: bash + + salt myminion git.list_worktrees /path/to/repo + salt myminion git.list_worktrees /path/to/repo all=True + salt myminion git.list_worktrees /path/to/repo stale=True + ''' + cwd = _expand_path(cwd, user) + kwargs = salt.utils.clean_kwargs(**kwargs) + all_ = kwargs.pop('all', False) + if kwargs: + salt.utils.invalid_kwargs(kwargs) + + if all_ and stale: + raise CommandExecutionError( + '\'all\' and \'stale\' cannot both be set to True' + ) + + try: + worktree_root = rev_parse(cwd, opts=['--git-path', 'worktrees']) + except CommandExecutionError as exc: + msg = 'Failed to find worktree location for ' + cwd + log.error(msg, exc_info_on_loglevel=logging.DEBUG) + raise CommandExecutionError(msg) + if worktree_root.startswith('.git'): + worktree_root = os.path.join(cwd, worktree_root) + if not os.path.isdir(worktree_root): + return {} + + worktree_info = {} + for worktree_name in os.listdir(worktree_root): + gitdir_file = os.path.join(worktree_root, worktree_name, 'gitdir') + try: + with open(gitdir_file, 'r') as fp_: + for line in fp_: + worktree_loc = line.rstrip('\n') + if worktree_loc.endswith('/.git'): + worktree_loc = worktree_loc[:-5] + worktree_info[worktree_name] = worktree_loc + break + except (IOError, OSError) as exc: + if exc.errno == errno.EEXIST: + log.warning( + gitdir_file + ' does not exist, data for worktree ' + + worktree_name + ' may be corrupted. Try pruning worktrees.' + ) + elif exc.errno == errno.EACCES: + raise CommandExecutionError( + 'Permission denied reading from ' + gitdir_file + ) + else: + raise CommandExecutionError( + 'Error {0} encountered reading from {1}: {2}'.format( + exc.errno, gitdir_file, exc.strerror + ) + ) + + if all_ or not worktree_info: + return worktree_info + + ret = {} + for worktree_name, worktree_loc in six.iteritems(worktree_info): + worktree_is_stale = not os.path.isdir(worktree_loc) + if (stale and worktree_is_stale) \ + or (not stale and not worktree_is_stale): + ret[worktree_name] = worktree_loc + return ret + + def ls_remote(cwd=None, remote='origin', ref='master', @@ -2400,7 +2501,7 @@ def reset(cwd, opts='', user=None, ignore_retcode=False): ignore_retcode=ignore_retcode)['stdout'] -def rev_parse(cwd, rev, opts='', user=None, ignore_retcode=False): +def rev_parse(cwd, rev=None, opts='', user=None, ignore_retcode=False): ''' .. versionadded:: 2015.8.0 @@ -2413,6 +2514,9 @@ def rev_parse(cwd, rev, opts='', user=None, ignore_retcode=False): Revision to parse. See the `SPECIFYING REVISIONS`_ section of the `git-rev-parse(1)`_ manpage for details on how to format this argument. + This argument is optional when using the options in the `Options for + Files` section of the `git-rev-parse(1)`_ manpage. + opts Any additional options to add to the command line, in a single string @@ -2424,10 +2528,9 @@ def rev_parse(cwd, rev, opts='', user=None, ignore_retcode=False): If ``True``, do not log an error to the minion log if the git command returns a nonzero exit status. - .. versionadded:: 2015.8.0 - .. _`git-rev-parse(1)`: http://git-scm.com/docs/git-rev-parse .. _`SPECIFYING REVISIONS`: http://git-scm.com/docs/git-rev-parse#_specifying_revisions + .. _`Options for Files`: http://git-scm.com/docs/git-rev-parse#_options_for_files CLI Examples: @@ -2442,11 +2545,16 @@ def rev_parse(cwd, rev, opts='', user=None, ignore_retcode=False): salt myminion git.rev_parse /path/to/repo 'develop@{upstream}' opts='--abbrev-ref' # Get the SHA1 for the commit corresponding to tag v1.2.3 salt myminion git.rev_parse /path/to/repo 'v1.2.3^{commit}' + # Find out whether or not the repo at /path/to/repo is a bare repository + salt myminion git.rev_parse /path/to/repo opts='--is-bare-repository' ''' cwd = _expand_path(cwd, user) command = ['git', 'rev-parse'] command.extend(_format_opts(opts)) - command.append(rev) + if rev is not None: + if not isinstance(rev, six.string_types): + rev = str(rev) + command.append(rev) return _git_run(command, cwd=cwd, runas=user, From 43fef21907148eadf206a8ff0ec75ff344d5cfeb Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 19 Aug 2015 19:09:59 -0500 Subject: [PATCH 10/13] Add list_worktrees tests --- tests/integration/modules/git.py | 85 ++++++++++++++++++++++++++++---- 1 file changed, 76 insertions(+), 9 deletions(-) diff --git a/tests/integration/modules/git.py b/tests/integration/modules/git.py index 785f0c643c..c23cff7b08 100644 --- a/tests/integration/modules/git.py +++ b/tests/integration/modules/git.py @@ -864,21 +864,24 @@ class GitModuleTest(integration.ModuleCase): @skipIf(not _worktrees_supported(), 'Git 2.5 or newer required for worktree support') - def test_worktree(self): + def test_worktrees(self): ''' - This tests git.worktree_add, git.is_worktree, git.worktree_rm, and - git.worktree_prune + This tests git.worktree_add, git.is_worktree, git.list_worktrees, + git.worktree_rm, and git.worktree_prune ''' - worktree_name = 'hotfix' worktree_path = tempfile.mkdtemp(dir=integration.TMP) worktree_basename = os.path.basename(worktree_path) - # Add a new worktree + worktree_path2 = tempfile.mkdtemp(dir=integration.TMP) + worktree_basename2 = os.path.basename(worktree_path2) + # Add the worktrees ret = self.run_function( - 'git.worktree_add', - [self.repo, worktree_path], - branch=worktree_name + 'git.worktree_add', [self.repo, worktree_path], ) self.assertTrue('Enter ' + worktree_path in ret) + ret = self.run_function( + 'git.worktree_add', [self.repo, worktree_path2] + ) + self.assertTrue('Enter ' + worktree_path2 in ret) # Check if this new path is a worktree self.assertTrue(self.run_function('git.is_worktree', [worktree_path])) # Check if the main repo is a worktree @@ -887,13 +890,56 @@ class GitModuleTest(integration.ModuleCase): empty_dir = tempfile.mkdtemp(dir=integration.TMP) self.assertFalse(self.run_function('git.is_worktree', [empty_dir])) shutil.rmtree(empty_dir) - # Remove the worktree + # Both worktrees should show up here + self.assertEqual( + self.run_function('git.list_worktrees', [self.repo]), + {os.path.basename(worktree_path): worktree_path, + os.path.basename(worktree_path2): worktree_path2} + ) + # There should be no stale worktrees right now + self.assertEqual( + self.run_function('git.list_worktrees', [self.repo], stale=True), + {} + ) + # Both worktrees should show in the all=True output + self.assertEqual( + self.run_function( + 'git.list_worktrees', + [self.repo], + **{'all': True} + ), + {os.path.basename(worktree_path): worktree_path, + os.path.basename(worktree_path2): worktree_path2} + ) + # Remove the first worktree self.assertTrue(self.run_function('git.worktree_rm', [worktree_path])) + # The first worktree should no longer show up here + self.assertEqual( + self.run_function('git.list_worktrees', [self.repo]), + {os.path.basename(worktree_path2): worktree_path2} + ) + # The first worktree should be identified as stale now + self.assertEqual( + self.run_function('git.list_worktrees', [self.repo], stale=True), + {os.path.basename(worktree_path): worktree_path} + ) + # Both worktrees should show in the all=True output + self.assertEqual( + self.run_function( + 'git.list_worktrees', + [self.repo], + **{'all': True} + ), + {os.path.basename(worktree_path): worktree_path, + os.path.basename(worktree_path2): worktree_path2} + ) # Prune the worktrees prune_message = ( 'Removing worktrees/{0}: gitdir file points to non-existent ' 'location'.format(worktree_basename) ) + # Test dry run output. It should match the same output we get when we + # actually prune the worktrees. self.assertEqual( self.run_function( 'git.worktree_prune', @@ -906,6 +952,27 @@ class GitModuleTest(integration.ModuleCase): self.run_function('git.worktree_prune', [self.repo]), prune_message ) + # The first worktree should still no longer show up here + self.assertEqual( + self.run_function('git.list_worktrees', [self.repo]), + {os.path.basename(worktree_path2): worktree_path2} + ) + # The first worktree should no loner be identified as stale, since it + # was just pruned. + self.assertEqual( + self.run_function('git.list_worktrees', [self.repo], stale=True), + {} + ) + # Only the second worktree should still show in the all=True output, + # since the first was pruned. + self.assertEqual( + self.run_function( + 'git.list_worktrees', + [self.repo], + **{'all': True} + ), + {os.path.basename(worktree_path2): worktree_path2} + ) if __name__ == '__main__': From 817078096f271d0c795ca354ff68236ded5ae45b Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 19 Aug 2015 19:10:17 -0500 Subject: [PATCH 11/13] Fix invalid :py:func: paths These are functions from the execution module, not the state module. --- doc/topics/releases/beryllium.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/doc/topics/releases/beryllium.rst b/doc/topics/releases/beryllium.rst index 84b9db5657..94a1912a4c 100644 --- a/doc/topics/releases/beryllium.rst +++ b/doc/topics/releases/beryllium.rst @@ -92,19 +92,19 @@ soon. New Functions in Git Execution Module ------------------------------------- -- :py:func:`git.config_get_regexp ` -- :py:func:`git.config_unset ` -- :py:func:`git.is_worktree ` -- :py:func:`git.list_branches ` -- :py:func:`git.list_tags ` -- :py:func:`git.list_worktrees ` -- :py:func:`git.merge_base ` -- :py:func:`git.merge_tree ` -- :py:func:`git.rev_parse ` -- :py:func:`git.version ` -- :py:func:`git.worktree_rm ` -- :py:func:`git.worktree_add ` -- :py:func:`git.worktree_prune ` +- :py:func:`git.config_get_regexp ` +- :py:func:`git.config_unset ` +- :py:func:`git.is_worktree ` +- :py:func:`git.list_branches ` +- :py:func:`git.list_tags ` +- :py:func:`git.list_worktrees ` +- :py:func:`git.merge_base ` +- :py:func:`git.merge_tree ` +- :py:func:`git.rev_parse ` +- :py:func:`git.version ` +- :py:func:`git.worktree_rm ` +- :py:func:`git.worktree_add ` +- :py:func:`git.worktree_prune ` Changes to Functions in Git Execution Module -------------------------------------------- From 9a87f052d587fd5a49298ef58399c44fd50017e8 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 20 Aug 2015 10:39:36 -0500 Subject: [PATCH 12/13] Update class name for old git pillar class --- tests/integration/modules/pillar.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/modules/pillar.py b/tests/integration/modules/pillar.py index 059f2edbe4..252d588128 100644 --- a/tests/integration/modules/pillar.py +++ b/tests/integration/modules/pillar.py @@ -107,9 +107,9 @@ class PillarModuleTest(integration.ModuleCase): 'cachedir': self.master_opts['cachedir'], } - git_pillar.LegacyGitPillar('master', original_url, opts) + git_pillar._LegacyGitPillar('master', original_url, opts) opts['ext_pillar'] = [{'git': 'master {0}'.format(changed_url)}] - grepo = git_pillar.LegacyGitPillar('master', changed_url, opts) + grepo = git_pillar._LegacyGitPillar('master', changed_url, opts) repo = git.Repo(rp_location) self.assertEqual(grepo.rp_location, repo.remotes.origin.url) @@ -123,7 +123,7 @@ class PillarModuleTest(integration.ModuleCase): pillar = self.run_function('pillar.data') for branch, env in [('dev', 'testing')]: - repo = git_pillar.LegacyGitPillar(branch, repo_url, self.master_opts) + repo = git_pillar._LegacyGitPillar(branch, repo_url, self.master_opts) self.assertIn(repo.working_dir, pillar['test_ext_pillar_opts']['pillar_roots'][env]) From cca92412d82ad76dfdae7420c949553acff60fc6 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Thu, 20 Aug 2015 18:51:44 -0500 Subject: [PATCH 13/13] Use salt.utils.fopen() instead of open() --- salt/modules/git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/git.py b/salt/modules/git.py index 6c4a2b48dd..c0190158ec 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -1607,7 +1607,7 @@ def list_worktrees(cwd, stale=False, user=None, **kwargs): for worktree_name in os.listdir(worktree_root): gitdir_file = os.path.join(worktree_root, worktree_name, 'gitdir') try: - with open(gitdir_file, 'r') as fp_: + with salt.utils.fopen(gitdir_file, 'r') as fp_: for line in fp_: worktree_loc = line.rstrip('\n') if worktree_loc.endswith('/.git'):