Merge branch 'develop' of github.com:saltstack/salt into develop

salt/modules/apt.py

@@ -110,34 +110,62 @@ def refresh_db():
     return servers
 
 
-def install(pkg, refresh=False, repo='', skip_verify=False,
+def install(name=None, refresh=False, repo='', skip_verify=False,
-            debconf=None, version=None, **kwargs):
+            debconf=None, pkgs=None, sources=None, **kwargs):
     '''
-    Install the passed package
+    Install the passed package, add refresh=True to update the dpkg database.
 
-    pkg
+    name
-        The name of the package to be installed
+        The name of the package to be installed. Note that this parameter is
-    refresh : False
+        ignored if either "pkgs" or "sources" is passed. Additionally, please
-        Update apt before continuing
+        note that this option can only be used to install packages from a
-    repo : (default)
+        software repository. To install a package file manually, use the
+        "sources" option.
+
+        CLI Example::
+            salt '*' pkg.install <package name>
+
+    refresh
+        Whether or not to refresh the package database before installing.
+
+    repo
         Specify a package repository to install from
         (e.g., ``apt-get -t unstable install somepackage``)
-    skip_verify : False
+
-        Skip the GPG verification check (e.g., ``--allow-unauthenticated``)
+    skip_verify
-    debconf : None
+        Skip the GPG verification check (e.g., ``--allow-unauthenticated``, or
+        ``--force-bad-verify`` for install from package file).
+
+    debconf
         Provide the path to a debconf answers file, processed before
         installation.
-    version : None
-        Install a specific version of the package, e.g. 1.0.9~ubuntu
 
-    Return a dict containing the new package names and versions::
+    version
+        Install a specific version of the package, e.g. 1.0.9~ubuntu. Ignored
+        if "pkgs" or "sources" is passed.
+
+
+    Multiple Package Installation Options:
+
+    pkgs
+        A list of packages to install from a software repository. Must be
+        passed as a python list.
+
+        CLI Example::
+            salt '*' pkg.install pkgs='["foo","bar"]'
+
+    sources
+        A list of RPM sources to use for installing the package(s) defined in
+        pkgs. Must be passed as a list of dicts.
+
+        CLI Example::
+            salt '*' pkg.install sources='[{"foo": "salt://foo.pkg.tar.xz"},{"bar": "salt://bar.pkg.tar.xz"}]'
+
+
+    Returns a dict containing the new package names and versions::
 
         {'<package>': {'old': '<old-version>',
-                'new': '<new-version>']}
+                       'new': '<new-version>']}
-
-    CLI Example::
-
-        salt '*' pkg.install <package name>
     '''
     # Note that this function will daemonize the subprocess
     # preventing a restart resulting from a salt-minion upgrade
@@ -150,37 +178,39 @@ def install(pkg, refresh=False, repo='', skip_verify=False,
     if debconf:
         __salt__['debconf.set_file'](debconf)
 
-    ret_pkgs = {}
+    pkg_params,pkg_type = __salt__['pkg_resource.parse_targets'](name,
-    old_pkgs = list_pkgs()
+                                                                 pkgs,
+                                                                 sources)
+    if pkg_params is None or len(pkg_params) == 0:
+        return {}
+    elif pkg_type == 'file':
+        if repo:
+            log.debug('Skipping "repo" option (invalid for package file '
+                      'installation).')
+        cmd = 'dpkg -i {verify} {pkg}'.format(
+            verify='--force-bad-verify' if skip_verify else '',
+            pkg=' '.join(pkg_params),
+        )
+    elif pkg_type == 'repository':
+        fname = ' '.join(pkg_params)
+        if len(pkg_params) == 1:
+            for vkey, vsign in (('eq', '='), ('version', '=')):
+                if kwargs.get(vkey) is not None:
+                    fname = '"{0}{1}{2}"'.format(fname, vsign, kwargs[vkey])
+                    break
+        cmd = 'apt-get -q -y {confold} {verify} {target} install {pkg}'.format(
+            confold='-o DPkg::Options::=--force-confold',
+            verify='--allow-unauthenticated' if skip_verify else '',
+            target='-t {0}'.format(repo) if repo else '',
+            pkg=fname,
+        )
 
-    if version:
+    old = list_pkgs()
-        pkg = '{0}={1}'.format(pkg, version)
+    stderr = __salt__['cmd.run_all'](cmd).get('stderr','')
-    elif 'eq' in kwargs:
+    if stderr:
-        pkg = '{0}={1}'.format(pkg, kwargs['eq'])
+        log.error(stderr)
-
+    new = list_pkgs()
-    kwargs = {
+    return __salt__['pkg_resource.find_changes'](old,new)
-        'pkg': pkg,
-        'target': ' -t {0}'.format(repo) if repo else '',
-        'confold': ' -o DPkg::Options::=--force-confold',
-        'verify': ' --allow-unauthenticated' if skip_verify else '',
-    }
-    cmd = 'apt-get -q -y {confold}{verify}{target} install {pkg}'.format(**kwargs)
-
-    __salt__['cmd.run'](cmd)
-    new_pkgs = list_pkgs()
-
-    for pkg in new_pkgs:
-        if pkg in old_pkgs:
-            if old_pkgs[pkg] == new_pkgs[pkg]:
-                continue
-            else:
-                ret_pkgs[pkg] = {'old': old_pkgs[pkg],
-                             'new': new_pkgs[pkg]}
-        else:
-            ret_pkgs[pkg] = {'old': '',
-                         'new': new_pkgs[pkg]}
-
-    return ret_pkgs
 
 
 def remove(pkg):

salt/modules/pacman.py

@@ -196,7 +196,9 @@ def install(name=None, refresh=False, pkgs=None, sources=None, **kwargs):
             cmd = 'pacman -S --noprogressbar --noconfirm {0}'.format(fname)
 
     old = list_pkgs()
-    __salt__['cmd.retcode'](cmd)
+    stderr = __salt__['cmd.run_all'](cmd).get('stderr','')
+    if stderr:
+        log.error(stderr)
     new = list_pkgs()
     return __salt__['pkg_resource.find_changes'](old,new)
 

salt/modules/pkg_resource.py

@@ -61,8 +61,22 @@ def _parse_pkg_meta(path):
         return name,version
 
     def parse_deb(path):
-        # Need to add support for this once apt gets some multi-source love
+        name = ''
-        pass
+        version = ''
+        result = __salt__['cmd.run_all']('dpkg-deb -I "{0}"'.format(path))
+        if result['retcode'] == 0:
+            for line in result['stdout'].splitlines():
+                if not name:
+                    m = re.match('^\s*Package\s*:\s*(\S+)',line)
+                    if m:
+                        name = m.group(1)
+                        continue
+                if not version:
+                    m = re.match('^\s*Version\s*:\s*(\S+)',line)
+                    if m:
+                        version = m.group(1)
+                        continue
+        return name,version
 
     if __grains__['os_family'] in ('Suse','RedHat','Mandriva'):
         metaparser = parse_rpm

salt/modules/yumpkg5.py

@@ -206,7 +206,9 @@ def install(name=None, refresh=False, repo='', skip_verify=False, pkgs=None,
         pkg=' '.join(pkg_params),
     )
     old = list_pkgs()
-    __salt__['cmd.retcode'](cmd)
+    stderr = __salt__['cmd.run_all'](cmd).get('stderr','')
+    if stderr:
+        log.error(stderr)
     new = list_pkgs()
     return __salt__['pkg_resource.find_changes'](old,new)
 

salt/modules/zypper.py

@@ -183,7 +183,9 @@ def install(name=None, refresh=False, pkgs=None, sources=None, **kwargs):
 
     cmd = 'zypper -n install -l {0}'.format(' '.join(pkg_params))
     old = list_pkgs()
-    __salt__['cmd.retcode'](cmd)
+    stderr = __salt__['cmd.run_all'](cmd).get('stderr','')
+    if stderr:
+        log.error(stderr)
     new = list_pkgs()
     return __salt__['pkg_resource.find_changes'](old,new)
 

salt/overstate.py

@@ -14,6 +14,7 @@ import sys
 
 # Import Salt libs
 import salt.client
+import salt.utils
 
 # Import third party libs
 import yaml
@@ -43,7 +44,7 @@ class OverState(object):
                     )
             if not os.path.isfile(fn_):
                 continue
-            with open(fn_) as fp_:
+            with salt.utils.fopen(fn_) as fp_:
                 try:
                     # TODO Use render system
                     return self.__sort_stages(yaml.load(fp_))

salt/states/virtualenv.py

@@ -55,13 +55,14 @@ def managed(name,
 
     # Bail out early if the specified requirements file can't be found
     if requirements:
-        reqs_hash = __salt__['cp.hash_file'](requirements, __env__)
+        orig_path = requirements
+        requirements = __salt__['cp.cache_file'](requirements, __env__)
 
-        if not reqs_hash:
+        if not requirements:
             ret.update({
                 'result': False,
                 'comment': "pip requirements file '{0}' not found".format(
-                    requirements)})
+                    orig_path)})
 
             return ret
 
@@ -110,6 +111,10 @@ def managed(name,
             requirements=requirements, bin_env=name, runas=runas, cwd=cwd
         )
         ret['result'] &= _ret['retcode']==0
+        if _ret['retcode'] > 0:
+            ret['comment'] = '{0}\n{1}\n{2}'.format(ret['comment'],
+                                                    _ret['stdout'],
+                                                    _ret['stderr'])
 
         after = set(__salt__['pip.freeze'](bin_env=name))
 

tests/integration/modules/cp.py

@@ -1,6 +1,9 @@
 # Import python libs
 import os
 import hashlib
+
+# Import salt libs
+import salt.utils
 import integration
 
 
@@ -19,7 +22,7 @@ class CPModuleTest(integration.ModuleCase):
                     'salt://grail/scene33',
                     tgt,
                 ])
-        with open(tgt, 'r') as scene:
+        with salt.utils.fopen(tgt, 'r') as scene:
             data = scene.read()
             self.assertIn('KNIGHT:  They\'re nervous, sire.', data)
             self.assertNotIn('bacon', data)
@@ -37,7 +40,7 @@ class CPModuleTest(integration.ModuleCase):
             ],
             template='jinja'
         )
-        with open(tgt, 'r') as cheese:
+        with salt.utils.fopen(tgt, 'r') as cheese:
             data = cheese.read()
             self.assertIn('Gromit', data)
             self.assertNotIn('bacon', data)
@@ -48,7 +51,7 @@ class CPModuleTest(integration.ModuleCase):
         '''
         tgt = os.path.join(integration.TMP, 'file.big')
         src = os.path.join(integration.FILES, 'file/base/file.big')
-        with open(src, 'r') as fp_:
+        with salt.utils.fopen(src, 'r') as fp_:
             hash = hashlib.md5(fp_.read()).hexdigest()
 
         self.run_function(
@@ -59,7 +62,7 @@ class CPModuleTest(integration.ModuleCase):
             ],
             gzip=5
         )
-        with open(tgt, 'r') as scene:
+        with salt.utils.fopen(tgt, 'r') as scene:
             data = scene.read()
             self.assertIn('KNIGHT:  They\'re nervous, sire.', data)
             self.assertNotIn('bacon', data)
@@ -78,7 +81,7 @@ class CPModuleTest(integration.ModuleCase):
             ],
             makedirs=True
         )
-        with open(tgt, 'r') as scene:
+        with salt.utils.fopen(tgt, 'r') as scene:
             data = scene.read()
             self.assertIn('KNIGHT:  They\'re nervous, sire.', data)
             self.assertNotIn('bacon', data)
@@ -95,7 +98,7 @@ class CPModuleTest(integration.ModuleCase):
                     tgt,
                     'spam=bacon',
                 ])
-        with open(tgt, 'r') as scene:
+        with salt.utils.fopen(tgt, 'r') as scene:
             data = scene.read()
             self.assertIn('bacon', data)
             self.assertNotIn('spam', data)
@@ -145,7 +148,7 @@ class CPModuleTest(integration.ModuleCase):
                     'salt://grail/scene33',
                     tgt,
                 ])
-        with open(tgt, 'r') as scene:
+        with salt.utils.fopen(tgt, 'r') as scene:
             data = scene.read()
             self.assertIn('KNIGHT:  They\'re nervous, sire.', data)
             self.assertNotIn('bacon', data)
@@ -159,7 +162,7 @@ class CPModuleTest(integration.ModuleCase):
                 [
                     'salt://grail/scene33',
                 ])
-        with open(ret, 'r') as scene:
+        with salt.utils.fopen(ret, 'r') as scene:
             data = scene.read()
             self.assertIn('KNIGHT:  They\'re nervous, sire.', data)
             self.assertNotIn('bacon', data)
@@ -174,7 +177,7 @@ class CPModuleTest(integration.ModuleCase):
                     ['salt://grail/scene33', 'salt://grail/36/scene'],
                 ])
         for path in ret:
-            with open(path, 'r') as scene:
+            with salt.utils.fopen(path, 'r') as scene:
                 data = scene.read()
                 self.assertIn('ARTHUR:', data)
                 self.assertNotIn('bacon', data)
@@ -194,12 +197,12 @@ class CPModuleTest(integration.ModuleCase):
         cp.cache_local_file
         '''
         src = os.path.join(integration.TMP, 'random')
-        with open(src, 'w+') as fn_:
+        with salt.utils.fopen(src, 'w+') as fn_:
             fn_.write('foo')
         ret = self.run_function(
                 'cp.cache_local_file',
                 [src])
-        with open(ret, 'r') as cp_:
+        with salt.utils.fopen(ret, 'r') as cp_:
             self.assertEqual(cp_.read(), 'foo')
 
     def test_list_states(self):
@@ -264,7 +267,7 @@ class CPModuleTest(integration.ModuleCase):
                 [
                     'salt://grail/scene33',
                 ])
-        with open(path, 'r') as fn_:
+        with salt.utils.fopen(path, 'r') as fn_:
             self.assertEqual(
                     md5_hash['hsum'],
                     hashlib.md5(fn_.read()).hexdigest()

tests/integration/modules/file.py

@@ -6,8 +6,9 @@ import shutil
 import sys
 
 # Import salt libs
-from saltunittest import skipIf
+import salt.utils
 import integration
+from saltunittest import skipIf
 
 
 class FileModuleTest(integration.ModuleCase):
@@ -16,7 +17,7 @@ class FileModuleTest(integration.ModuleCase):
     '''
     def setUp(self):
         self.myfile = os.path.join(integration.TMP, 'myfile')
-        with open(self.myfile, 'w+') as fp:
+        with salt.utils.fopen(self.myfile, 'w+') as fp:
             fp.write("Hello\n")
         self.mydir = os.path.join(integration.TMP, 'mydir/isawesome')
         if not os.path.isdir(self.mydir):
@@ -42,7 +43,6 @@ class FileModuleTest(integration.ModuleCase):
         self.assertEqual(fstat.st_uid, os.getuid())
         self.assertEqual(fstat.st_gid, grp.getgrnam(group).gr_gid)
 
-
     @skipIf(sys.platform.startswith('win'), 'No chgrp on Windows')
     def test_chown_no_user(self):
         user = 'notanyuseriknow'
@@ -79,7 +79,6 @@ class FileModuleTest(integration.ModuleCase):
         self.assertEqual(fstat.st_uid, os.getuid())
         self.assertEqual(fstat.st_gid, os.getgid())
 
-
     @skipIf(sys.platform.startswith('win'), 'No chgrp on Windows')
     def test_chgrp(self):
         if sys.platform == 'darwin':
@@ -104,18 +103,18 @@ class FileModuleTest(integration.ModuleCase):
         src_patch = os.path.join(
             integration.FILES, 'file', 'base', 'hello.patch')
         src_file = os.path.join(integration.TMP, 'src.txt')
-        with open(src_file, 'w+') as fp:
+        with salt.utils.fopen(src_file, 'w+') as fp:
             fp.write("Hello\n")
 
         # dry-run should not modify src_file
         ret = self.minion_run('file.patch', src_file, src_patch, dry_run=True)
         assert ret['retcode'] == 0, repr(ret)
-        with open(src_file) as fp:
+        with salt.utils.fopen(src_file) as fp:
             self.assertEqual(fp.read(), 'Hello\n')
 
         ret = self.minion_run('file.patch', src_file, src_patch)
         assert ret['retcode'] == 0, repr(ret)
-        with open(src_file) as fp:
+        with salt.utils.fopen(src_file) as fp:
             self.assertEqual(fp.read(), 'Hello world\n')
 
     def test_remove_file(self):

tests/integration/modules/hosts.py

@@ -1,10 +1,12 @@
 '''
 Test the hosts module
 '''
+# Import python libs
 import os
 import shutil
 
 # Import Salt libs
+import salt.utils
 import integration
 
 HFN = os.path.join(integration.TMP, 'hosts')
@@ -143,7 +145,7 @@ class HostsModuleTest(integration.ModuleCase):
         # use an empty one so we can prove the syntax of the entries
         # being added by the hosts module
         self.__clear_hosts()
-        f = open(HFN, 'w')
+        f = salt.utils.fopen(HFN, 'w')
         f.close()
 
         assert self.run_function(
@@ -166,7 +168,7 @@ class HostsModuleTest(integration.ModuleCase):
         )
 
         # now read the lines and ensure they're formatted correctly
-        lines = open(HFN, 'r').readlines()
+        lines = salt.utils.fopen(HFN, 'r').readlines()
         self.assertEqual(lines, [
             "192.168.1.3\t\thost3.fqdn.com\n",
             "192.168.1.2\t\thost2.fqdn.com\thost2\toldhost2\thost2-reorder\n",

tests/integration/modules/ssh.py

@@ -6,9 +6,9 @@ import os
 import shutil
 
 # Import Salt libs
+import salt.utils
 import integration
 
-
 AUTHORIZED_KEYS = os.path.join('/tmp/subsalttest', 'authorized_keys')
 KNOWN_HOSTS = os.path.join('/tmp/subsalttest', 'known_hosts')
 GITHUB_FINGERPRINT = '16:27:ac:a5:76:28:2d:36:63:1b:56:4d:eb:df:a6:48'
@@ -23,7 +23,8 @@ class SSHModuleTest(integration.ModuleCase):
         Set up the ssh module tests
         '''
         super(SSHModuleTest, self).setUp()
-        with open(os.path.join(integration.FILES, 'ssh', 'raw')) as fd:
+        ssh_raw_path = os.path.join(integration.FILES, 'ssh', 'raw')
+        with salt.utils.fopen(ssh_raw_path) as fd:
             self.key = fd.read().strip()
 
     def tearDown(self):

tests/integration/modules/state.py

@@ -1,6 +1,9 @@
 # Import python libs
 import os
 import shutil
+
+# Import salt libs
+import salt.utils
 import integration
 
 
@@ -100,7 +103,7 @@ fi
 if [ -f /etc/bash_completion ] && ! shopt -oq posix; then
     . /etc/bash_completion
 fi
-''', open(testfile, 'r').read())
+''', salt.utils.fopen(testfile, 'r').read())
 
         # Re-append switching order
         ret = self.run_function('state.sls', mods='testappend.step-2')
@@ -135,7 +138,7 @@ fi
 if [ -f /etc/bash_completion ] && ! shopt -oq posix; then
     . /etc/bash_completion
 fi
-''', open(testfile, 'r').read())
+''', salt.utils.fopen(testfile, 'r').read())
 
     def test_issue_1876_syntax_error(self):
         '''
@@ -215,7 +218,7 @@ fi
         try:
             self.assertMultiLineEqual(
                 contents,
-                open(testfile, 'r').read()
+                salt.utils.fopen(testfile, 'r').read()
             )
             # Make sure we don't re-append existing text
             ret = self.run_function('state.sls', mods='issue-1879.step-1')
@@ -238,7 +241,7 @@ fi
                 raise
             self.assertMultiLineEqual(
                 contents,
-                open(testfile, 'r').read()
+                salt.utils.fopen(testfile, 'r').read()
             )
         except Exception:
             if os.path.exists(testfile):
@@ -312,7 +315,7 @@ fi
             'files', 'file', 'base', 'issue-2068-template-str-no-dot.sls'
         )
 
-        template = open(template_path, 'r').read()
+        template = salt.utils.fopen(template_path, 'r').read()
         try:
             ret = self.run_function('state.template_str', [template])
 
@@ -360,7 +363,7 @@ fi
             'files', 'file', 'base', 'issue-2068-template-str.sls'
         )
 
-        template = open(template_path, 'r').read()
+        template = salt.utils.fopen(template_path, 'r').read()
         try:
             ret = self.run_function('state.template_str', [template])
 

tests/integration/shell/cp.py

@@ -14,9 +14,9 @@ import yaml
 import pipes
 
 # Import salt libs
-from saltunittest import TestLoader, TextTestRunner
+import salt.utils
 import integration
-from integration import TestDaemon
+from saltunittest import TestLoader, TextTestRunner
 
 
 class CopyTest(integration.ShellCase, integration.ShellCaseCommonTestsMixIn):
@@ -42,7 +42,7 @@ class CopyTest(integration.ShellCase, integration.ShellCaseCommonTestsMixIn):
                 'files', 'file', 'base', 'testfile'
             )
         )
-        testfile_contents = open(testfile, 'r').read()
+        testfile_contents = salt.utils.fopen(testfile, 'r').read()
 
         for idx, minion in enumerate(minions):
             ret = self.run_salt(
@@ -107,6 +107,6 @@ if __name__ == "__main__":
     loader = TestLoader()
     tests = loader.loadTestsFromTestCase(CopyTest)
     print('Setting up Salt daemons to execute tests')
-    with TestDaemon():
+    with integration.TestDaemon():
         runner = TextTestRunner(verbosity=1).run(tests)
         sys.exit(runner.wasSuccessful())

tests/integration/states/file.py

@@ -8,6 +8,7 @@ import shutil
 
 # Import salt libs
 import integration
+import salt.utils
 
 
 class FileTest(integration.ModuleCase):
@@ -40,7 +41,7 @@ class FileTest(integration.ModuleCase):
         file.absent
         '''
         name = os.path.join(integration.TMP, 'file_to_kill')
-        with open(name, 'w+') as fp_:
+        with salt.utils.fopen(name, 'w+') as fp_:
             fp_.write('killme')
         ret = self.run_state('file.absent', name=name)
         result = self.state_result(ret)
@@ -79,7 +80,7 @@ class FileTest(integration.ModuleCase):
         file.absent test interface
         '''
         name = os.path.join(integration.TMP, 'file_to_kill')
-        with open(name, 'w+') as fp_:
+        with salt.utils.fopen(name, 'w+') as fp_:
             fp_.write('killme')
         ret = self.run_state('file.absent', test=True, name=name)
         result = self.state_result(ret)
@@ -98,9 +99,9 @@ class FileTest(integration.ModuleCase):
         src = os.path.join(
             integration.FILES, 'file', 'base', 'grail', 'scene33'
         )
-        with open(src, 'r') as fp_:
+        with salt.utils.fopen(src, 'r') as fp_:
             master_data = fp_.read()
-        with open(name, 'r') as fp_:
+        with salt.utils.fopen(name, 'r') as fp_:
             minion_data = fp_.read()
         self.assertEqual(master_data, minion_data)
         result = self.state_result(ret)
@@ -173,7 +174,9 @@ class FileTest(integration.ModuleCase):
             template='jinja', defaults={'spam': _ts})
         result = self.state_result(ret)
         self.assertTrue(result)
-        self.assertIn(_ts, open(os.path.join(name, 'scene33'), 'r').read())
+        self.assertIn(
+            _ts, salt.utils.fopen(os.path.join(name, 'scene33'), 'r').read()
+        )
         shutil.rmtree(name, ignore_errors=True)
 
     def test_recurse_clean(self):
@@ -183,9 +186,9 @@ class FileTest(integration.ModuleCase):
         name = os.path.join(integration.TMP, 'recurse_clean_dir')
         os.makedirs(name)
         strayfile = os.path.join(name, 'strayfile')
-        open(strayfile, 'w').close()
+        salt.utils.fopen(strayfile, 'w').close()
         # Corner cases: replacing file with a directory and vice versa
-        open(os.path.join(name, '36'), 'w').close()
+        salt.utils.fopen(os.path.join(name, '36'), 'w').close()
         os.makedirs(os.path.join(name, 'scene33'))
         ret = self.run_state(
             'file.recurse', name=name, source='salt://grail', clean=True)
@@ -201,12 +204,12 @@ class FileTest(integration.ModuleCase):
         file.sed
         '''
         name = os.path.join(integration.TMP, 'sed_test')
-        with open(name, 'w+') as fp_:
+        with salt.utils.fopen(name, 'w+') as fp_:
             fp_.write('change_me')
         ret = self.run_state(
             'file.sed', name=name, before='change', after='salt'
         )
-        with open(name, 'r') as fp_:
+        with salt.utils.fopen(name, 'r') as fp_:
             self.assertIn('salt', fp_.read())
         result = self.state_result(ret)
         self.assertTrue(result)
@@ -217,12 +220,12 @@ class FileTest(integration.ModuleCase):
         file.sed test integration
         '''
         name = os.path.join(integration.TMP, 'sed_test_test')
-        with open(name, 'w+') as fp_:
+        with salt.utils.fopen(name, 'w+') as fp_:
             fp_.write('change_me')
         ret = self.run_state(
             'file.sed', test=True, name=name, before='change', after='salt'
         )
-        with open(name, 'r') as fp_:
+        with salt.utils.fopen(name, 'r') as fp_:
             self.assertIn('change', fp_.read())
         result = self.state_result(ret)
         self.assertIsNone(result)
@@ -234,12 +237,12 @@ class FileTest(integration.ModuleCase):
         '''
         name = os.path.join(integration.TMP, 'comment_test')
         # write a line to file
-        with open(name, 'w+') as fp_:
+        with salt.utils.fopen(name, 'w+') as fp_:
             fp_.write('comment_me')
         # comment once
         _ret = self.run_state('file.comment', name=name, regex='^comment')
         # line is commented
-        with open(name, 'r') as fp_:
+        with salt.utils.fopen(name, 'r') as fp_:
             self.assertTrue(fp_.read().startswith('#comment'))
         # result is positive
         ret = list(_ret.values())[0]
@@ -247,7 +250,7 @@ class FileTest(integration.ModuleCase):
         # comment twice
         _ret = self.run_state('file.comment', name=name, regex='^comment')
         # line is still commented
-        with open(name, 'r') as fp_:
+        with salt.utils.fopen(name, 'r') as fp_:
             self.assertTrue(fp_.read().startswith('#comment'))
         # result is still positive
         ret = list(_ret.values())[0]
@@ -259,12 +262,12 @@ class FileTest(integration.ModuleCase):
         file.comment test interface
         '''
         name = os.path.join(integration.TMP, 'comment_test_test')
-        with open(name, 'w+') as fp_:
+        with salt.utils.fopen(name, 'w+') as fp_:
             fp_.write('comment_me')
         ret = self.run_state(
             'file.comment', test=True, name=name, regex='.*comment.*',
         )
-        with open(name, 'r') as fp_:
+        with salt.utils.fopen(name, 'r') as fp_:
             self.assertNotIn('#comment', fp_.read())
         result = self.state_result(ret)
         self.assertIsNone(result)
@@ -275,10 +278,10 @@ class FileTest(integration.ModuleCase):
         file.uncomment
         '''
         name = os.path.join(integration.TMP, 'uncomment_test')
-        with open(name, 'w+') as fp_:
+        with salt.utils.fopen(name, 'w+') as fp_:
             fp_.write('#comment_me')
         ret = self.run_state('file.uncomment', name=name, regex='^comment')
-        with open(name, 'r') as fp_:
+        with salt.utils.fopen(name, 'r') as fp_:
             self.assertNotIn('#comment', fp_.read())
         result = self.state_result(ret)
         self.assertTrue(result)
@@ -289,12 +292,12 @@ class FileTest(integration.ModuleCase):
         file.comment test interface
         '''
         name = os.path.join(integration.TMP, 'uncomment_test_test')
-        with open(name, 'w+') as fp_:
+        with salt.utils.fopen(name, 'w+') as fp_:
             fp_.write('#comment_me')
         ret = self.run_state(
             'file.uncomment', test=True, name=name, regex='^comment.*'
         )
-        with open(name, 'r') as fp_:
+        with salt.utils.fopen(name, 'r') as fp_:
             self.assertIn('#comment', fp_.read())
         result = self.state_result(ret)
         self.assertIsNone(result)
@@ -305,10 +308,10 @@ class FileTest(integration.ModuleCase):
         file.append
         '''
         name = os.path.join(integration.TMP, 'append_test')
-        with open(name, 'w+') as fp_:
+        with salt.utils.fopen(name, 'w+') as fp_:
             fp_.write('#salty!')
         ret = self.run_state('file.append', name=name, text='cheese')
-        with open(name, 'r') as fp_:
+        with salt.utils.fopen(name, 'r') as fp_:
             self.assertIn('cheese', fp_.read())
         result = self.state_result(ret)
         self.assertTrue(result)
@@ -319,10 +322,10 @@ class FileTest(integration.ModuleCase):
         file.append test interface
         '''
         name = os.path.join(integration.TMP, 'append_test_test')
-        with open(name, 'w+') as fp_:
+        with salt.utils.fopen(name, 'w+') as fp_:
             fp_.write('#salty!')
         ret = self.run_state('file.append', test=True, name=name, text='cheese')
-        with open(name, 'r') as fp_:
+        with salt.utils.fopen(name, 'r') as fp_:
             self.assertNotIn('cheese', fp_.read())
         result = self.state_result(ret)
         self.assertIsNone(result)
@@ -429,7 +432,7 @@ class FileTest(integration.ModuleCase):
             for change in ret.values():
                 self.assertTrue(isinstance(change, dict))
                 self.assertTrue(change['result'])
-            contents = open(tmp_file_append, 'r').read()
+            contents = salt.utils.fopen(tmp_file_append, 'r').read()
 
             # It should not append text again
             ret = self.run_function(
@@ -439,7 +442,9 @@ class FileTest(integration.ModuleCase):
                 self.assertTrue(isinstance(change, dict))
                 self.assertTrue(change['result'])
 
-            self.assertEqual(contents, open(tmp_file_append, 'r').read())
+            self.assertEqual(
+                contents, salt.utils.fopen(tmp_file_append, 'r').read()
+            )
 
         except AssertionError:
             shutil.copy(tmp_file_append, tmp_file_append + '.bak')
@@ -452,7 +457,7 @@ class FileTest(integration.ModuleCase):
         if not self.run_function('cmd.has_exec', ['patch']):
             self.skipTest('patch is not installed')
         src_file = os.path.join(integration.TMP, 'src.txt')
-        with open(src_file, 'w+') as fp:
+        with salt.utils.fopen(src_file, 'w+') as fp:
             fp.write(src)
         ret = self.run_state('file.patch',
             name=src_file,
@@ -464,7 +469,7 @@ class FileTest(integration.ModuleCase):
     def test_patch(self):
         src_file, ret = self.do_patch()
         self.assert_success(ret)
-        with open(src_file) as fp:
+        with salt.utils.fopen(src_file) as fp:
             self.assertEqual(fp.read(), 'Hello world\n')
 
     def test_patch_hash_mismatch(self):
@@ -484,7 +489,7 @@ class FileTest(integration.ModuleCase):
         # Get a path to the temporary file
         tmp_file = os.path.join(integration.TMP, 'issue-2041-comment.txt')
         # Write some data to it
-        open(tmp_file, 'w').write('hello\nworld\n')
+        salt.utils.fopen(tmp_file, 'w').write('hello\nworld\n')
         # create the sls template
         template_lines = [
             "{0}:".format(tmp_file),
@@ -529,7 +534,7 @@ class FileTest(integration.ModuleCase):
         # Get a path to the temporary file
         tmp_file = os.path.join(integration.TMP, 'issue-2379-file-append.txt')
         # Write some data to it
-        open(tmp_file, 'w').write(
+        salt.utils.fopen(tmp_file, 'w').write(
             'hello\nworld\n' +          # Some junk
             '#PermitRootLogin yes\n' +  # Commented text
             '# PermitRootLogin yes\n'   # Commented text with space

tests/integration/states/host.py

@@ -5,8 +5,9 @@ tests for host state
 # Import python libs
 import os
 import shutil
-#
+
 # Import salt libs
+import salt.utils
 import integration
 
 HFILE = os.path.join(integration.TMP, 'hosts')
@@ -35,7 +36,7 @@ class HostTest(integration.ModuleCase):
         ret = self.run_state('host.present', name=name, ip=ip)
         result = self.state_result(ret)
         self.assertTrue(result)
-        with open(HFILE) as fp_:
+        with salt.utils.fopen(HFILE) as fp_:
             output = fp_.read()
             self.assertIn('{0}\t\t{1}'.format(ip, name), output)
 

tests/integration/states/match.py

@@ -5,11 +5,16 @@
     :license: Apache 2.0, see LICENSE for more details
 """
 
+# Import python libs
 import os
+
+# Import salt libs
+import salt.utils
 import integration
 
 STATE_DIR = os.path.join(integration.FILES, 'file', 'base')
 
+
 class StateMatchTest(integration.ModuleCase):
     '''
     Validate the file state
@@ -27,7 +32,7 @@ class StateMatchTest(integration.ModuleCase):
         top_filename = 'issue-2167-ipcidr-match.sls'
         top_file = os.path.join(STATE_DIR, top_filename)
         try:
-            open(top_file, 'w').write(
+            salt.utils.fopen(top_file, 'w').write(
                 'base:\n'
                 '  {0}:\n'
                 '    - match: ipcidr\n'
@@ -35,7 +40,8 @@ class StateMatchTest(integration.ModuleCase):
             )
             ret = self.run_function('state.top', [top_filename])
             self.assertNotIn(
-                "AttributeError: 'Matcher' object has no attribute 'functions'",
+                'AttributeError: \'Matcher\' object has no attribute '
+                '\'functions\'',
                 ret
             )
         finally:

tests/unit/config_test.py

@@ -7,9 +7,13 @@
     :license: Apache 2.0, see LICENSE for more details.
 """
 
+# Import python libs
 import os
 import shutil
 import tempfile
+
+# Import salt libs
+import salt.utils
 from saltunittest import TestCase, TestLoader, TextTestRunner
 from salt import config as sconfig
 
@@ -17,7 +21,7 @@ from salt import config as sconfig
 class ConfigTestCase(TestCase):
     def test_proper_path_joining(self):
         fpath = tempfile.mktemp()
-        open(fpath, 'w').write(
+        salt.utils.fopen(fpath, 'w').write(
             "root_dir: /\n"
             "key_logfile: key\n"
         )
@@ -32,7 +36,7 @@ class ConfigTestCase(TestCase):
         root_dir = os.path.join(tempdir, 'foo', 'bar')
         os.makedirs(root_dir)
         fpath = os.path.join(root_dir, 'config')
-        open(fpath, 'w').write(
+        salt.utils.fopen(fpath, 'w').write(
             'root_dir: {0}\n'
             'log_file: {1}\n'.format(root_dir, fpath)
         )

tests/unit/templates/jinja_test.py

@@ -1,9 +1,14 @@
+# Import python libs
 import os
 import tempfile
+
+# Import 3rd party libs
 from jinja2 import Environment
+
+# Import salt libs
+import salt.utils
 from salt.utils.jinja import SaltCacheLoader
 from salt.utils.templates import render_jinja_tmpl
-
 from saltunittest import TestCase
 
 TEMPLATES_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -111,7 +116,7 @@ class TestGetTemplate(TestCase):
         if the file is not contained in the searchpath
         '''
         fn_ = os.path.join(TEMPLATES_DIR, 'files', 'test', 'hello_simple')
-        with open(fn_) as fp_:
+        with salt.utils.fopen(fn_) as fp_:
             out = render_jinja_tmpl(
                     fp_.read(),
                     dict(opts=self.local_opts, env='other'))
@@ -124,7 +129,7 @@ class TestGetTemplate(TestCase):
         '''
         filename = os.path.join(TEMPLATES_DIR, 'files', 'test', 'hello_import')
         out = render_jinja_tmpl(
-                open(filename).read(),
+                salt.utils.fopen(filename).read(),
                 dict(opts=self.local_opts, env='other'))
         self.assertEqual(out, 'Hey world !a b !')
 
@@ -141,7 +146,7 @@ class TestGetTemplate(TestCase):
         SaltCacheLoader.file_client = lambda loader: fc
         filename = os.path.join(TEMPLATES_DIR, 'files', 'test', 'hello_import')
         out = render_jinja_tmpl(
-                open(filename).read(),
+                salt.utils.fopen(filename).read(),
                 dict(opts={'cachedir': TEMPLATES_DIR, 'file_client': 'remote'},
                      a='Hi', b='Salt', env='test'))
         self.assertEqual(out, 'Hey world !Hi Salt !')

tests/unit/utils/find_test.py

@@ -1,11 +1,14 @@
+# Import python libs
 import os
 import sys
 import shutil
 import tempfile
 import stat
-from saltunittest import TestCase, TestLoader, TextTestRunner, skipIf
 
+# Import salt libs
+import salt.utils
 import salt.utils.find
+from saltunittest import TestCase, TestLoader, TextTestRunner, skipIf
 
 
 class TestFind(TestCase):
@@ -276,7 +279,7 @@ class TestGrepOption(TestCase):
 
     def test_grep_option_match_regular_file(self):
         hello_file = os.path.join(self.tmpdir, 'hello.txt')
-        fd = open(hello_file, 'w')
+        fd = salt.utils.fopen(hello_file, 'w')
         fd.write("foo")
         fd.close()
         option = salt.utils.find.GrepOption('grep', 'foo')
@@ -336,7 +339,7 @@ class TestPrintOption(TestCase):
 
     def test_print_option_execute(self):
         hello_file = os.path.join(self.tmpdir, 'hello.txt')
-        fd = open(hello_file, 'w')
+        fd = salt.utils.fopen(hello_file, 'w')
         fd.write("foo")
         fd.close()
 
@@ -526,7 +529,7 @@ class TestFinder(TestCase):
 
     def test_find(self):
         hello_file = os.path.join(self.tmpdir, 'hello.txt')
-        fd = open(hello_file, 'w')
+        fd = salt.utils.fopen(hello_file, 'w')
         fd.write("foo")
         fd.close()
 

tests/unit/utils/verify_test.py

@@ -12,6 +12,7 @@ import resource
 import tempfile
 
 # Import Salt libs
+import salt.utils
 from saltunittest import skipIf, TestCase, TestsLoggingHandler
 
 from salt.utils.verify import (
@@ -114,7 +115,8 @@ class TestVerify(TestCase):
                                       (127, 'WARNING'), (196, 'CRITICAL')):
 
                     for n in range(prev, newmax):
-                        with open(os.path.join(keys_dir, str(n)), 'w') as fp_:
+                        kpath = os.path.join(keys_dir, str(n))
+                        with salt.utils.fopen(kpath, 'w') as fp_:
                             fp_.write(str(n))
 
                     opts = {
@@ -148,7 +150,8 @@ class TestVerify(TestCase):
 
                 newmax = mof_test
                 for n in range(prev, newmax):
-                    with open(os.path.join(keys_dir, str(n)), 'w') as fp_:
+                    kpath = os.path.join(keys_dir, str(n))
+                    with salt.utils.fopen(kpath, 'w') as fp_:
                         fp_.write(str(n))
 
                 opts = {