Merge pull request #9015 from cowyn/develop

Added exception handler for git repo init

salt/fileserver/gitfs.py

@@ -127,7 +127,14 @@ def init():
         rp_ = os.path.join(bp_, repo_hash)
         if not os.path.isdir(rp_):
             os.makedirs(rp_)
-        repo = git.Repo.init(rp_)
+
+        try:
+            repo = git.Repo.init(rp_)
+        except Exception as e:
+            log.error('GitPython exception caught while initializing the repo '
+                      'for gitfs: {0}. Maybe git is not available.'.format(e))
+            return repos
+
         if not repo.remotes:
             try:
                 repo.create_remote('origin', opt)

salt/master.py

@@ -53,6 +53,7 @@ import salt.utils.gzip_util
 from salt.utils.debug import enable_sigusr1_handler, inspect_stack
 from salt.exceptions import SaltMasterError, MasterExit
 from salt.utils.event import tagify
+from salt.pillar import git_pillar
 
 # Import halite libs
 try:
@@ -191,6 +192,14 @@ class Master(SMaster):
         runners = salt.loader.runner(self.opts)
         schedule = salt.utils.schedule.Schedule(self.opts, runners)
         ckminions = salt.utils.minions.CkMinions(self.opts)
+
+        pillargitfs = None
+        for opts_dict in [x for x in self.opts.get('ext_pillar', [])]:
+            if 'git' in opts_dict:
+                br, loc = opts_dict['git'].strip().split()
+                pillargitfs = git_pillar.GitPillar(br, loc, self.opts)
+                break
+
         while True:
             now = int(time.time())
             loop_interval = int(self.opts['loop_interval'])
@@ -223,7 +232,7 @@ class Master(SMaster):
                     search.index()
             try:
                 if not fileserver.servers:
-                    log.error('No fileservers loaded, The master will not be'
+                    log.error('No fileservers loaded, the master will not be'
                               'able to serve files to minions')
                     raise SaltMasterError('No fileserver backends available')
                 fileserver.update()
@@ -231,6 +240,12 @@ class Master(SMaster):
                 log.error(
                     'Exception {0} occurred in file server update'.format(exc)
                 )
+            try:
+                if pillargitfs is not None:
+                    pillargitfs.update()
+            except Exception as exc:
+                log.error('Exception {0} occurred in file server update '
+                          'for git_pillar module.'.format(exc))
             try:
                 schedule.eval()
                 # Check if scheduler requires lower loop interval than

salt/pillar/git_pillar.py

@@ -80,91 +80,124 @@ def __virtual__():
     return __virtualname__
 
 
-def _get_ref(repo, short):
+class GitPillar(object):
     '''
-    Return bool if the short ref is in the repo
+    Deal with the remote git repository for Pillar
     '''
-    for ref in repo.refs:
-        if isinstance(ref, git.RemoteReference):
-            parted = ref.name.partition('/')
-            refname = parted[2] if parted[2] else parted[0]
-            if short == refname:
-                return ref
-    return False
+
+    def __init__(self, branch, repo_location, opts):
+        '''
+        Try to initilize the Git repo object
+        '''
+        self.br = branch
+        self.rp_location = repo_location
+        self.opts = opts
+        self.envs = set()
+        self.working_dir = ''
+        self.repo = None
+
+        for idx, opts_dict in enumerate(self.opts['ext_pillar']):
+            if opts_dict.get('git', '') == '{0} {1}'.format(self.br,
+                                                            self.rp_location):
+                rp_ = os.path.join(self.opts['cachedir'],
+                                   'pillar_gitfs', str(idx))
+
+                if not os.path.isdir(rp_):
+                    os.makedirs(rp_)
+
+                try:
+                    self.repo = git.Repo.init(rp_)
+                except Exception as e:
+                    log.error('GitPython exception caught while '
+                              'initializing the repo: {0}. Maybe '
+                              'git is not available.'.format(e))
+
+                # Git directory we are working on
+                # Should be the same as self.repo.working_dir
+                self.working_dir = rp_
+
+                if isinstance(self.repo, git.Repo):
+                    if not self.repo.remotes:
+                        try:
+                            self.repo.create_remote('origin', self.rp_location)
+                            # ignore git ssl verification if requested
+                            if self.opts.get('pillar_gitfs_ssl_verify', True):
+                                self.repo.git.config('http.sslVerify', 'true')
+                            else:
+                                self.repo.git.config('http.sslVerify', 'false')
+                        except os.error:
+                            # This exception occurs when two processes are trying to write
+                            # to the git config at once, go ahead and pass over it since
+                            # this is the only write
+                            # This should place a lock down
+                            pass
+
+                break
 
 
-def init(branch, repo_location):
-    '''
-    Return the git repo object for this session
-    '''
-    # get index
-    ind = None
-    for index, opts_dict in enumerate(__opts__['ext_pillar']):
-        if opts_dict.get('git', '') == '{0} {1}'.format(branch, repo_location):
-            ind = index
-            break
+    def update(self):
+        '''
+        Ensure you are following the latest changes on the remote
 
-    if ind is None:
-        return None
-
-    rp_ = os.path.join(__opts__['cachedir'], 'pillar_gitfs', str(ind))
-
-    if not os.path.isdir(rp_):
-        os.makedirs(rp_)
-    repo = git.Repo.init(rp_)
-    if not repo.remotes:
+        Return boolean wether it worked
+        '''
         try:
-            repo.create_remote('origin', repo_location)
-            # ignore git ssl verification if requested
-            if __opts__.get('pillar_gitfs_ssl_verify', True):
-                repo.git.config('http.sslVerify', 'true')
-            else:
-                repo.git.config('http.sslVerify', 'false')
-        except os.error:
-            # This exception occurs when two processes are trying to write
-            # to the git config at once, go ahead and pass over it since
-            # this is the only write
-            # This should place a lock down
-            pass
-    repo.git.fetch()
-    return repo
+            log.debug('Updating fileserver for git_pillar module')
+            self.repo.git.fetch()
+        except git.exc.GitCommandError as e:
+            log.error('Unable to fetch the latest changes from remote '
+                      '{0}: {1}'.format(self.rp_location, e))
+            return False
+
+        try:
+            self.repo.git.checkout("origin/" + self.br)
+        except git.exc.GitCommandError as e:
+            logging.error('Unable to checkout branch '
+                          '{0}: {1}'.format(self.br, e))
+            return False
+
+        return True
+
+
+    def envs(self):
+        '''
+        Return a list of refs that can be used as environments
+        '''
+
+        if isinstance(self.repo, git.Repo):
+            remote = self.repo.remote()
+            for ref in self.repo.refs:
+                parted = ref.name.partition('/')
+                short = parted[2] if parted[2] else parted[0]
+                if isinstance(ref, git.Head):
+                    if short == 'master':
+                        short = 'base'
+                    if ref not in remote.stale_refs:
+                        self.envs.add(short)
+                elif isinstance(ref, git.Tag):
+                    self.envs.add(short)
+
+        return list(self.envs)
 
 
 def update(branch, repo_location):
     '''
-    Ensure you are on the right branch, and execute a git pull
+    Ensure you are following the latest changes on the remote
 
     return boolean whether it worked
     '''
-    pid = os.getpid()
-    repo = init(branch, repo_location)
-    try:
-        repo.git.checkout("origin/" + branch)
-    except git.exc.GitCommandError as e:
-        logging.error('Unable to checkout branch {0}: {1}'.format(branch, e))
-        return False
-    return True
+    gitpil = GitPillar(branch, repo_location, __opts__)
+
+    return gitpil.update()
 
 
 def envs(branch, repo_location):
     '''
     Return a list of refs that can be used as environments
     '''
-    ret = set()
-    repo = init(branch, repo_location)
+    gitpil = GitPillar(branch, repo_location, __opts__)
 
-    remote = repo.remote()
-    for ref in repo.refs:
-        parted = ref.name.partition('/')
-        short = parted[2] if parted[2] else parted[0]
-        if isinstance(ref, git.Head):
-            if short == 'master':
-                short = 'base'
-            if ref not in remote.stale_refs:
-                ret.add(short)
-        elif isinstance(ref, git.Tag):
-            ret.add(short)
-    return list(ret)
+    return gitpil.envs()
 
 
 def ext_pillar(minion_id, pillar, repo_string):
@@ -174,26 +207,19 @@ def ext_pillar(minion_id, pillar, repo_string):
     # split the branch and repo name
     branch, repo_location = repo_string.strip().split()
 
+    gitpil = GitPillar(branch, repo_location, __opts__)
+
     # environment is "different" from the branch
-    branch_env = branch
-    if branch_env == 'master':
-        branch_env = 'base'
-
-    # Update first
-    if not update(branch, repo_location):
-        return {}
-
-    # get the repo
-    repo = init(branch, repo_location)
+    branch = (branch == 'master' and 'base' or branch)
 
     # Don't recurse forever-- the Pillar object will re-call the ext_pillar
     # function
-    if __opts__['pillar_roots'].get(branch_env, []) == [repo.working_dir]:
+    if __opts__['pillar_roots'].get(branch, []) == [gitpil.working_dir]:
         return {}
 
     opts = deepcopy(__opts__)
 
-    opts['pillar_roots'][branch_env] = [repo.working_dir]
+    opts['pillar_roots'][branch] = [gitpil.working_dir]
 
     pil = Pillar(opts, __grains__, minion_id, 'base')
 

salt/runners/git_pillar.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+'''
+Directly manage the salt git_pillar plugin
+'''
+
+# Import salt libs
+import salt.pillar.git_pillar
+
+
+def update(branch, repo):
+    '''
+    Execute an update for the configured git fileserver backend for Pillar
+
+    CLI Example:
+
+    .. code-block:: bash
+
+        salt-run git_pillar.update branch='branch' repo='location'
+    '''
+    fileserver = salt.pillar.git_pillar.GitPillar(branch, repo, __opts__)
+    fileserver.update()