Merge branch '2017.7' into '2018.3'

Conflicts:
  - requirements/dev_python27.txt
  - requirements/dev_python34.txt
  - salt/fileclient.py
  - salt/output/__init__.py
  - salt/serializers/configparser.py
  - salt/states/pkg.py
  - salt/states/zpool.py
  - salt/transport/tcp.py
  - tests/integration/cloud/providers/test_ec2.py
  - tests/integration/modules/test_mac_user.py
  - tests/integration/modules/test_pip.py
  - tests/integration/modules/test_pkg.py
  - tests/integration/modules/test_service.py
  - tests/integration/states/test_pip_state.py
  - tests/integration/states/test_user.py
  - tests/unit/modules/test_cmdmod.py

.gitignore

@@ -11,6 +11,8 @@ MANIFEST
 *.wpr
 *.wpu
 *.DS_Store
+.pytest_cache
+Pipfile.lock
 
 # virtualenv
 # - ignores directories of a virtualenv when you create it right on

Gemfile

@@ -2,8 +2,8 @@
 
 source 'https://rubygems.org'
 
-gem 'test-kitchen', :git => 'https://github.com/gtmanfred/test-kitchen.git'
-gem 'kitchen-salt', :git => 'https://github.com/saltstack/kitchen-salt.git'
+gem 'test-kitchen', '~>1.21'
+gem 'kitchen-salt', '~>0.2'
 gem 'kitchen-sync'
 gem 'git'
 
@@ -20,7 +20,7 @@ group :windows do
   gem 'vagrant-wrapper'
   gem 'kitchen-vagrant'
   gem 'winrm', '~>2.0'
-  gem 'winrm-fs', :git => 'https://github.com/gtmanfred/winrm-fs.git'
+  gem 'winrm-fs', :git => 'https://github.com/WinRb/winrm-fs.git'
 end
 
 group :ec2 do

Pipfile

@@ -0,0 +1,40 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+Jinja2 = "*"
+msgpack-python = ">0.3,!=0.5.5"
+PyYAML = "*"
+MarkupSafe = "*"
+requests = ">=1.0.0"
+tornado = ">=4.2.1,<5.0"
+pycrypto = ">=2.6.1"
+pyzmq = ">=2.2.0"
+
+[dev-packages]
+mock = ">=2.0.0"
+apache-libcloud = ">=0.14.0"
+boto = ">=2.32.1"
+boto3 = ">=1.2.1"
+moto = ">=0.3.6"
+SaltPyLint = ">=v2017.3.6"
+pytest = ">=3.5.0"
+
+[packages.futures]
+# Required by Tornado to handle threads stuff.
+version = ">=2.0"
+markers = "python_version < '3.0'"
+
+[dev-packages.pytest-salt]
+git = "git://github.com/saltstack/pytest-salt.git"
+ref = "master"
+
+[dev-packages.httpretty]
+# httpretty Needs to be here for now even though it's a dependency of boto.
+# A pip install on a fresh system will decide to target httpretty 0.8.10 to
+# satisfy other requirements, and httpretty 0.8.10 has bugs in setup.py that
+# prevent it from being successfully installed (at least on Python 3.4).
+version = "*"
+markers = "python_version >= '3.4'"

doc/topics/cloud/linode.rst

@@ -44,7 +44,7 @@ at ``/etc/salt/cloud.profiles`` or in the ``/etc/salt/cloud.profiles.d/`` direct
 
     linode_1024:
       provider: my-linode-config
-      size: Linode 2048
+      size: Linode 2GB
       image: CentOS 7
       location: London, England, UK
 
@@ -77,12 +77,14 @@ command:
         ----------
         linode:
             ----------
-            Linode 1024:
+            Linode 2GB:
                 ----------
                 AVAIL:
                     ----------
                     10:
                         500
+                    11:
+                        500
                     2:
                         500
                     3:
@@ -100,11 +102,19 @@ command:
                 CORES:
                     1
                 DISK:
-                    24
+                    50
                 HOURLY:
                     0.015
                 LABEL:
-                    Linode 1024
+                    Linode 2GB
+                PLANID:
+                    2
+                PRICE:
+                    10.0
+                RAM:
+                    2048
+                XFER:
+                    2000
     ...SNIP...
 
 

pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+addopts = --ssh-tests -ra -sv
+testpaths = tests
+norecursedirs = tests/kitchen

requirements/base-py2.txt

@@ -1,4 +0,0 @@
--r base.txt
-
-# Required by Tornado to handle threads stuff.
-futures>=2.0

requirements/base-py3.txt

@@ -1 +0,0 @@
--r base.txt

requirements/base.txt

@@ -7,4 +7,4 @@ MarkupSafe
 requests>=1.0.0
 tornado>=4.2.1,<6.0
 # Required by Tornado to handle threads stuff.
-futures>=2.0
+futures>=2.0; python_version < '3.0'

requirements/dev.txt

@@ -0,0 +1,17 @@
+-r base.txt
+
+mock>=2.0.0
+apache-libcloud>=0.14.0
+boto>=2.32.1
+boto3>=1.2.1
+moto>=0.3.6
+SaltPyLint>=v2017.3.6
+pytest>=3.5.0
+git+https://github.com/saltstack/pytest-salt.git@master#egg=pytest-salt
+testinfra>=1.7.0
+
+# httpretty Needs to be here for now even though it's a dependency of boto.
+# A pip install on a fresh system will decide to target httpretty 0.8.10 to
+# satisfy other requirements, and httpretty 0.8.10 has bugs in setup.py that
+# prevent it from being successfully installed (at least on Python 3.4).
+httpretty; python_version >= '3.4'

requirements/dev_python27.txt

@@ -1,12 +1,2 @@
--r base-py2.txt
-
-mock>=2.0.0
-apache-libcloud>=0.14.0
-boto>=2.32.1
-boto3>=1.2.1
-moto>=0.3.6
-SaltPyLint>=v2017.3.6
-pytest>=3.5.0
-git+https://github.com/eisensheng/pytest-catchlog.git@develop#egg=Pytest-catchlog
-git+https://github.com/saltstack/pytest-salt.git@master#egg=pytest-salt
-testinfra>=1.7.0
+# This is a legacy file, use dev.txt
+-r dev.txt

requirements/dev_python34.txt

@@ -1,17 +1,2 @@
--r base-py3.txt
-
-mock>=2.0.0
-apache-libcloud>=0.14.0
-boto>=2.32.1
-boto3>=1.2.1
-moto>=0.3.6
-# httpretty Needs to be here for now even though it's a dependency of boto.
-# A pip install on a fresh system will decide to target httpretty 0.8.10 to
-# satisfy other requirements, and httpretty 0.8.10 has bugs in setup.py that
-# prevent it from being successfully installed (at least on Python 3.4).
-httpretty
-SaltPyLint>=v2017.2.29
-pytest>=3.5.0
-git+https://github.com/saltstack/pytest-salt.git@master#egg=pytest-salt
-git+https://github.com/eisensheng/pytest-catchlog.git@develop#egg=Pytest-catchlog
-testinfra>=1.7.0
+# This is a legacy file, use dev.txt
+-r dev.txt

salt/cloud/__init__.py

@@ -1916,7 +1916,7 @@ class Map(Cloud):
         pmap = self.map_providers_parallel(cached=cached)
         exist = set()
         defined = set()
-        for profile_name, nodes in six.iteritems(self.rendered_map):
+        for profile_name, nodes in six.iteritems(copy.deepcopy(self.rendered_map)):
             if profile_name not in self.opts['profiles']:
                 msg = (
                     'The required profile, \'{0}\', defined in the map '
@@ -1937,13 +1937,13 @@ class Map(Cloud):
             # Get associated provider data, in case something like size
             # or image is specified in the provider file. See issue #32510.
             alias, driver = profile_data.get('provider').split(':')
-            provider_details = self.opts['providers'][alias][driver].copy()
+            provider_details = copy.deepcopy(self.opts['providers'][alias][driver])
             del provider_details['profiles']
 
             # Update the provider details information with profile data
             # Profile data should override provider data, if defined.
             # This keeps map file data definitions consistent with -p usage.
-            provider_details.update(profile_data)
+            salt.utils.dictupdate.update(provider_details, profile_data)
             profile_data = provider_details
 
             for nodename, overrides in six.iteritems(nodes):

salt/fileclient.py

@@ -47,6 +47,7 @@ from salt.ext.six.moves.urllib.parse import urlparse, urlunparse
 # pylint: enable=no-name-in-module,import-error
 
 log = logging.getLogger(__name__)
+MAX_FILENAME_LENGTH = 255
 
 
 def get_file_client(opts, pillar=False):
@@ -831,6 +832,9 @@ class Client(object):
         else:
             file_name = url_data.path
 
+        if len(file_name) > MAX_FILENAME_LENGTH:
+            file_name = salt.utils.hashutils.sha256_digest(file_name)
+
         return salt.utils.path.join(
             cachedir,
             'extrn_files',

salt/modules/boto3_route53.py

@@ -646,6 +646,10 @@ def disassociate_vpc_from_hosted_zone(HostedZoneId=None, Name=None, VPCId=None,
             r = conn.disassociate_vpc_from_hosted_zone(**args)
             return _wait_for_sync(r['ChangeInfo']['Id'], conn)
         except ClientError as e:
+            if e.response.get('Error', {}).get('Code') == 'VPCAssociationNotFound':
+                log.debug('No VPC Association exists.')
+                # return True since the current state is the desired one
+                return True
             if tries and e.response.get('Error', {}).get('Code') == 'Throttling':
                 log.debug('Throttled by AWS API.')
                 time.sleep(3)

salt/modules/boto_secgroup.py

@@ -473,6 +473,9 @@ def authorize(name=None, source_group_name=None,
                 log.error(msg)
                 return False
         except boto.exception.EC2ResponseError as e:
+            # if we are trying to add the same rule then we are already in the desired state, return true
+            if e.error_code == 'InvalidPermission.Duplicate':
+                return True
             msg = ('Failed to add rule to security group {0} with id {1}.'
                    .format(group.name, group.id))
             log.error(msg)

salt/modules/mac_service.py

@@ -409,7 +409,7 @@ def enabled(name, runas=None):
         return False
 
 
-def disabled(name, runas=None):
+def disabled(name, runas=None, domain='system'):
     '''
     Check if the specified service is not enabled. This is the opposite of
     ``service.enabled``
@@ -418,6 +418,8 @@ def disabled(name, runas=None):
 
     :param str runas: User to run launchctl commands
 
+    :param str domain: domain to check for disabled services. Default is system.
+
     :return: True if the specified service is NOT enabled, otherwise False
     :rtype: bool
 
@@ -427,8 +429,22 @@ def disabled(name, runas=None):
 
         salt '*' service.disabled org.cups.cupsd
     '''
-    # A service is disabled if it is not enabled
-    return not enabled(name, runas=runas)
+    ret = False
+    disabled = launchctl('print-disabled',
+                         domain,
+                         return_stdout=True,
+                         output_loglevel='trace',
+                         runas=runas)
+    for service in disabled.split("\n"):
+        if name in service:
+            srv_name = service.split("=>")[0].split("\"")[1]
+            status = service.split("=>")[1]
+            if name != srv_name:
+                pass
+            else:
+                return True if 'true' in status.lower() else False
+
+    return False
 
 
 def get_all(runas=None):

salt/modules/win_system.py

@@ -511,7 +511,7 @@ def get_system_info():
 
     .. code-block:: bash
 
-        salt 'minion-id' system.get_info
+        salt 'minion-id' system.get_system_info
     '''
     os_type = {1: 'Work Station',
                2: 'Domain Controller',

salt/modules/yumpkg.py

@@ -92,6 +92,7 @@ def _strip_headers(output, *args):
     if not args:
         args_lc = ('installed packages',
                    'available packages',
+                   'available upgrades',
                    'updated packages',
                    'upgraded packages')
     else:

salt/modules/zypper.py

@@ -853,7 +853,7 @@ def _get_configured_repos():
     '''
 
     repos_cfg = configparser.ConfigParser()
-    repos_cfg.read([REPOS + '/' + fname for fname in os.listdir(REPOS)])
+    repos_cfg.read([REPOS + '/' + fname for fname in os.listdir(REPOS) if fname.endswith(".repo")])
 
     return repos_cfg
 

salt/serializers/configparser.py

@@ -85,15 +85,28 @@ def serialize(obj, **options):
         raise SerializationError(error)
 
 
-def _read_dict(configparser, dictionary):
+def _is_defaultsect(section_name):
+    if six.PY3:
+        return section_name == configparser.DEFAULTSECT
+    else:  # in py2 the check is done against lowercased section name
+        return section_name.upper() == configparser.DEFAULTSECT
+
+
+def _read_dict(cp, dictionary):
     '''
     Cribbed from python3's ConfigParser.read_dict function.
     '''
     for section, keys in dictionary.items():
         section = six.text_type(section)
-        configparser.add_section(section)
+
+        if _is_defaultsect(section):
+            if six.PY2:
+                section = configparser.DEFAULTSECT
+        else:
+            cp.add_section(section)
+
         for key, value in keys.items():
-            key = configparser.optionxform(six.text_type(key))
+            key = cp.optionxform(six.test_type(key))
             if value is not None:
                 value = six.text_type(value)
-            configparser.set(section, key, value)
+            cp.set(section, key, value)

salt/state.py

@@ -261,11 +261,21 @@ def find_sls_ids(sls, high):
     '''
     ret = []
     for nid, item in six.iteritems(high):
-        if item['__sls__'] == sls:
-            for st_ in item:
-                if st_.startswith('__'):
-                    continue
-                ret.append((nid, st_))
+        try:
+            sls_tgt = item['__sls__']
+        except TypeError:
+            if nid != '__exclude__':
+                log.error(
+                    'Invalid non-dict item \'%s\' in high data. Value: %r',
+                    nid, item
+                )
+            continue
+        else:
+            if sls_tgt == sls:
+                for st_ in item:
+                    if st_.startswith('__'):
+                        continue
+                    ret.append((nid, st_))
     return ret
 
 

salt/states/zpool.py

@@ -311,15 +311,20 @@ def present(name, properties=None, filesystem_properties=None, layout=None, conf
 
         ## NOTE: build list of properties to update
         properties_update = []
-        for prop in properties:
-            ## NOTE: skip unexisting properties
-            if prop not in properties_current:
-                log.warning('zpool.present::%s::update - unknown property: %s', name, prop)
-                continue
+        if properties:
+            for prop in properties:
+                ## NOTE: skip unexisting properties
+                if prop not in properties_current:
+                    log.warning('zpool.present::%s::update - unknown property: %s', name, prop)
+                    continue
 
-            ## NOTE: compare current and wanted value
-            if properties_current[prop] != properties[prop]:
-                properties_update.append(prop)
+                value = properties[prop]
+                if isinstance(value, bool):
+                    value = 'on' if value else 'off'
+
+                ## NOTE: compare current and wanted value
+                if properties_current[prop] != value:
+                    properties_update.append(prop)
 
         ## NOTE: update pool properties
         for prop in properties_update:

tests/conftest.py

@@ -103,6 +103,14 @@ def pytest_addoption(parser):
              'SSH server on your machine. In certain environments, this '
              'may be insecure! Default: False'
     )
+    test_selection_group.addoption(
+        '--proxy',
+        '--proxy-tests',
+        dest='proxy',
+        action='store_true',
+        default=False,
+        help='Run proxy tests'
+    )
     test_selection_group.addoption(
         '--run-destructive',
         action='store_true',
@@ -641,7 +649,8 @@ def test_daemon(request):
               ('sysinfo', request.config.getoption('--sysinfo')),
               ('no_colors', request.config.getoption('--no-colors')),
               ('output_columns', request.config.getoption('--output-columns')),
-              ('ssh', request.config.getoption('--ssh')))
+              ('ssh', request.config.getoption('--ssh')),
+              ('proxy', request.config.getoption('--proxy')))
     options = namedtuple('options', [n for n, v in values])(*[v for n, v in values])
     fake_parser = namedtuple('parser', 'options')(options)
 

tests/integration/cloud/providers/test_ec2.py

@@ -10,27 +10,27 @@ import yaml
 
 # Import Salt Libs
 from salt.config import cloud_providers_config
+import salt.utils.cloud
 import salt.utils.files
 
 # Import Salt Testing Libs
 from tests.support.case import ShellCase
 from tests.support.paths import FILES
 from tests.support.helpers import expensiveTest, generate_random_name
-from tests.support.unit import expectedFailure
+from tests.support.unit import expectedFailure, skipIf
 from tests.support import win_installer
 
 # Create the cloud instance name to be used throughout the tests
 INSTANCE_NAME = generate_random_name('CLOUD-TEST-')
 PROVIDER_NAME = 'ec2'
-
-EC2_TIMEOUT = 1000
+HAS_WINRM = salt.utils.cloud.HAS_WINRM and salt.utils.cloud.HAS_SMB
+TIMEOUT = 1000
 
 
 class EC2Test(ShellCase):
     '''
     Integration tests for the EC2 cloud provider in Salt-Cloud
     '''
-    TIMEOUT = 1000
 
     def _installer_name(self):
         '''
@@ -172,19 +172,17 @@ class EC2Test(ShellCase):
         '''
         # create the instance
         rename = INSTANCE_NAME + '-rename'
-        instance = self.run_cloud('-p ec2-test {0} --no-deploy'.format(INSTANCE_NAME),
-                                  timeout=EC2_TIMEOUT)
+        instance = self.run_cloud('-p ec2-test {0} --no-deploy'.format(INSTANCE_NAME), timeout=TIMEOUT)
         ret_str = '{0}:'.format(INSTANCE_NAME)
 
         # check if instance returned
         try:
             self.assertIn(ret_str, instance)
         except AssertionError:
-            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME),
-                           timeout=EC2_TIMEOUT)
+            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=TIMEOUT)
             raise
 
-        change_name = self.run_cloud('-a rename {0} newname={1} --assume-yes'.format(INSTANCE_NAME, rename), timeout=EC2_TIMEOUT)
+        change_name = self.run_cloud('-a rename {0} newname={1} --assume-yes'.format(INSTANCE_NAME, rename), timeout=TIMEOUT)
 
         check_rename = self.run_cloud('-a show_instance {0} --assume-yes'.format(rename), [rename])
         exp_results = ['        {0}:'.format(rename), '            size:',
@@ -193,13 +191,11 @@ class EC2Test(ShellCase):
             for result in exp_results:
                 self.assertIn(result, check_rename[0])
         except AssertionError:
-            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME),
-                           timeout=EC2_TIMEOUT)
+            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=TIMEOUT)
             raise
 
         # delete the instance
-        delete = self.run_cloud('-d {0} --assume-yes'.format(rename),
-                                timeout=EC2_TIMEOUT)
+        delete = self.run_cloud('-d {0} --assume-yes'.format(rename), timeout=TIMEOUT)
         ret_str = '                    shutting-down'
 
         # check if deletion was performed appropriately
@@ -227,8 +223,9 @@ class EC2Test(ShellCase):
                 'win_installer': self.copy_file(self.INSTALLER),
             },
         )
-        self._test_instance('ec2-win2012r2-test', debug=True, timeout=500)
+        self._test_instance('ec2-win2012r2-test', debug=True, timeout=TIMEOUT)
 
+    @skipIf(not HAS_WINRM, 'Skip when winrm dependencies are missing')
     def test_win2012r2_winrm(self):
         '''
         Tests creating and deleting a Windows 2012r2 instance on EC2 using
@@ -240,10 +237,11 @@ class EC2Test(ShellCase):
                 'userdata_file': self.copy_file('windows-firewall.ps1'),
                 'win_installer': self.copy_file(self.INSTALLER),
                 'winrm_ssl_verify': False,
+                'use_winrm': True,
             }
 
         )
-        self._test_instance('ec2-win2012r2-test', debug=True, timeout=500)
+        self._test_instance('ec2-win2012r2-test', debug=True, timeout=TIMEOUT)
 
     @expectedFailure
     def test_win2016_winexe(self):
@@ -261,8 +259,9 @@ class EC2Test(ShellCase):
                 'win_installer': self.copy_file(self.INSTALLER),
             },
         )
-        self._test_instance('ec2-win2016-test', debug=True, timeout=500)
+        self._test_instance('ec2-win2016-test', debug=True, timeout=TIMEOUT)
 
+    @skipIf(not HAS_WINRM, 'Skip when winrm dependencies are missing')
     def test_win2016_winrm(self):
         '''
         Tests creating and deleting a Windows 2016 instance on EC2 using winrm
@@ -274,10 +273,11 @@ class EC2Test(ShellCase):
                 'userdata_file': self.copy_file('windows-firewall.ps1'),
                 'win_installer': self.copy_file(self.INSTALLER),
                 'winrm_ssl_verify': False,
+                'use_winrm': True,
             }
 
         )
-        self._test_instance('ec2-win2016-test', debug=True, timeout=500)
+        self._test_instance('ec2-win2016-test', debug=True, timeout=TIMEOUT)
 
     def tearDown(self):
         '''
@@ -288,4 +288,4 @@ class EC2Test(ShellCase):
 
         # if test instance is still present, delete it
         if ret_str in query:
-            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=self.TIMEOUT)
+            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=TIMEOUT)

tests/integration/files/conf/cloud.profiles.d/ec2.conf

@@ -1,7 +1,7 @@
 ec2-test:
   provider: ec2-config
   image: ami-98aa1cf0
-  size: t1.micro
+  size: m1.small
   sh_username: ec2-user
   script_args: '-P -Z'
 ec2-win2012r2-test:
@@ -16,7 +16,6 @@ ec2-win2012r2-test:
   userdata_template: False
   use_winrm: True
   winrm_verify_ssl: False
-  ssh_interface: private_ips
   deploy: True
 ec2-win2016-test:
   provider: ec2-config
@@ -30,5 +29,4 @@ ec2-win2016-test:
   userdata_template: False
   use_winrm: True
   winrm_verify_ssl: False
-  ssh_interface: private_ips
   deploy: True

tests/integration/files/conf/cloud.profiles.d/linode.conf

@@ -1,5 +1,5 @@
 linode-test:
   provider: linode-config
-  size: Linode 2048
+  size: Linode 2GB
   image: Ubuntu 14.04 LTS
   script_args: '-P -Z'

tests/integration/files/file/base/issue-47182/slsfile1.sls

@@ -0,0 +1,2 @@
+slsfile1-nop:
+  test.nop

tests/integration/files/file/base/issue-47182/slsfile2.sls

@@ -0,0 +1,2 @@
+slsfile2-nop:
+  test.nop

tests/integration/files/file/base/issue-47182/stateA/init.sls

@@ -0,0 +1,2 @@
+include:
+  - issue-47182.stateA.newer

tests/integration/files/file/base/issue-47182/stateA/newer.sls

@@ -0,0 +1,6 @@
+exclude:
+  - sls: issue-47182.stateA
+
+somestuff:
+  cmd.run:
+    - name: echo This supersedes the stuff previously done in issue-47182.stateA

tests/integration/files/file/base/issue-47182/stateB.sls

@@ -0,0 +1,10 @@
+include:
+  - issue-47182.slsfile1
+  - issue-47182.slsfile2
+
+some-state:
+  test.nop:
+    - require:
+      - sls: issue-47182.slsfile1
+    - require_in:
+      - sls: issue-47182.slsfile2

tests/integration/files/file/base/issue-47182/top.sls

@@ -0,0 +1,4 @@
+base:
+  '*':
+    - issue-47182.stateA
+    - issue-47182.stateB

tests/integration/modules/test_mac_user.py

@@ -16,6 +16,7 @@ from tests.support.helpers import destructiveTest, skip_if_not_root
 # Import Salt Libs
 import salt.utils.files
 from salt.exceptions import CommandExecutionError
+import salt.ext.six as six
 
 # Import 3rd-party libs
 from salt.ext.six.moves import range  # pylint: disable=import-error,redefined-builtin

tests/integration/modules/test_pkg.py

@@ -335,16 +335,28 @@ class PkgModuleTest(ModuleCase, SaltReturnAssertsMixin):
         check that pkg.latest_version returns the latest version of the uninstalled package (it does not install the package, just checking the version)
         '''
         grains = self.run_function('grains.items')
-        cmd_info = self.run_function('pkg.info_installed', ['htop'])
-        if cmd_info != 'ERROR: package htop is not installed':
-            cmd_remove = self.run_function('pkg.remove', ['htop'])
+        remove = False
+        if salt.utils.is_windows():
+            cmd_info = self.run_function('pkg.version', [self.pkg])
+            remove = False if cmd_info == '' else True
+        else:
+            cmd_info = self.run_function('pkg.info_installed', [self.pkg])
+            if cmd_info != 'ERROR: package {0} is not installed'.format(self.pkg):
+                remove = True
+
+        # remove package if its installed
+        if remove:
+            cmd_remove = self.run_function('pkg.remove', [self.pkg])
+
         if grains['os_family'] == 'RedHat':
-            cmd_htop = self.run_function('cmd.run', ['yum list htop'])
+            cmd_pkg = self.run_function('cmd.run', ['yum list {0}'.format(self.pkg)])
+        elif salt.utils.is_windows():
+            cmd_pkg = self.run_function('pkg.list_available', [self.pkg])
         elif grains['os_family'] == 'Debian':
-            cmd_htop = self.run_function('cmd.run', ['apt list htop'])
+            cmd_pkg = self.run_function('cmd.run', ['apt list {0}'.format(self.pkg)])
         elif grains['os_family'] == 'Arch':
-            cmd_htop = self.run_function('cmd.run', ['pacman -Si htop'])
+            cmd_pkg = self.run_function('cmd.run', ['pacman -Si {0}'.format(self.pkg)])
         elif grains['os_family'] == 'Suse':
-            cmd_htop = self.run_function('cmd.run', ['zypper info htop'])
-        pkg_latest = self.run_function('pkg.latest_version', ['htop'])
-        self.assertIn(pkg_latest, cmd_htop)
+            cmd_pkg = self.run_function('cmd.run', ['zypper info {0}'.format(self.pkg)])
+        pkg_latest = self.run_function('pkg.latest_version', [self.pkg])
+        self.assertIn(pkg_latest, cmd_pkg)

tests/integration/modules/test_service.py

@@ -11,6 +11,7 @@ from tests.support.unit import skipIf
 # Import Salt libs
 import salt.utils.path
 import salt.utils.platform
+import salt.utils.systemd
 
 
 @destructiveTest
@@ -100,7 +101,38 @@ class ServiceModuleTest(ModuleCase):
         self.assertTrue(self.run_function('service.enable', [self.service_name]))
 
         self.assertTrue(self.run_function('service.disable', [self.service_name]))
-        self.assertIn(self.service_name, self.run_function('service.get_disabled'))
+        if salt.utils.platform.is_darwin():
+            self.assertTrue(self.run_function('service.disabled', [self.service_name]))
+        else:
+            self.assertIn(self.service_name, self.run_function('service.get_disabled'))
+
+    def test_service_disable_doesnot_exist(self):
+        '''
+        test service.get_disabled and service.disable module
+        when service name does not exist
+        '''
+        # enable service before test
+        srv_name = 'doesnotexist'
+        enable = self.run_function('service.enable', [srv_name])
+        systemd = salt.utils.systemd.booted()
+
+        # check service was not enabled
+        if systemd:
+            self.assertIn('ERROR', enable)
+        else:
+            self.assertFalse(enable)
+
+        # check service was not disabled
+        if tuple(self.run_function('grains.item', ['osrelease_info'])['osrelease_info']) == (14, 0o4) and not systemd:
+            # currently upstart does not have a mechanism to report if disabling a service fails if does not exist
+            self.assertTrue(self.run_function('service.disable', [srv_name]))
+        else:
+            self.assertFalse(self.run_function('service.disable', [srv_name]))
+
+        if salt.utils.platform.is_darwin():
+            self.assertFalse(self.run_function('service.disabled', [srv_name]))
+        else:
+            self.assertNotIn(srv_name, self.run_function('service.get_disabled'))
 
     @skipIf(not salt.utils.platform.is_windows(), 'Windows Only Test')
     def test_service_get_service_name(self):

tests/integration/states/test_user.py

@@ -44,12 +44,6 @@ class UserTest(ModuleCase, SaltReturnAssertsMixin):
     user_name = 'salt_test'
     user_home = '/var/lib/salt_test'
 
-    def setUp(self):
-        if salt.utils.platform.is_darwin():
-            #on mac we need to add user, because there is
-            #no creationtime for nobody user.
-            add_user = self.run_function('user.add', [USER], gid=GID)
-
     def test_user_absent(self):
         ret = self.run_state('user.absent', name='unpossible')
         self.assertSaltTrueReturn(ret)

tests/support/paths.py

@@ -50,6 +50,8 @@ SYS_TMP_DIR = os.path.abspath(os.path.realpath(
 ))
 TMP = os.path.join(SYS_TMP_DIR, 'salt-tests-tmpdir')
 FILES = os.path.join(INTEGRATION_TEST_DIR, 'files')
+BASE_FILES = os.path.join(FILES, 'file', 'base')
+PROD_FILES = os.path.join(FILES, 'file', 'prod')
 PYEXEC = 'python{0}.{1}'.format(*sys.version_info)
 MOCKBIN = os.path.join(INTEGRATION_TEST_DIR, 'mockbin')
 SCRIPT_DIR = os.path.join(CODE_DIR, 'scripts')

tests/unit/modules/test_mac_service.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+'''
+    :codeauthor: :email:`Megan Wilhite<mwilhite@saltstack.com>`
+'''
+
+# Import Python libs
+from __future__ import absolute_import
+
+# Import Salt Libs
+import salt.modules.mac_service as mac_service
+
+# Import Salt Testing Libs
+from tests.support.mixins import LoaderModuleMockMixin
+from tests.support.unit import skipIf, TestCase
+from tests.support.mock import (
+    MagicMock,
+    patch,
+    NO_MOCK,
+    NO_MOCK_REASON
+)
+
+
+@skipIf(NO_MOCK, NO_MOCK_REASON)
+class MacServiceTestCase(TestCase, LoaderModuleMockMixin):
+    '''
+    TestCase for salt.modules.mac_service module
+    '''
+    def setup_loader_modules(self):
+        return {mac_service: {}}
+
+    def test_service_disabled_when_enabled(self):
+        '''
+        test service.disabled when service is enabled
+        '''
+        srv_name = 'com.apple.atrun'
+        cmd = 'disabled services = {\n\t"com.saltstack.salt.minion" => false\n\t"com.apple.atrun" => false\n{'
+
+        with patch.object(mac_service, 'launchctl', MagicMock(return_value=cmd)):
+            self.assertFalse(mac_service.disabled(srv_name))
+
+    def test_service_disabled_when_disabled(self):
+        '''
+        test service.disabled when service is disabled
+        '''
+        srv_name = 'com.apple.atrun'
+        cmd = 'disabled services = {\n\t"com.saltstack.salt.minion" => false\n\t"com.apple.atrun" => true\n{'
+
+        with patch.object(mac_service, 'launchctl', MagicMock(return_value=cmd)):
+            self.assertTrue(mac_service.disabled(srv_name))
+
+    def test_service_disabled_srvname_wrong(self):
+        '''
+        test service.disabled when service is just slightly wrong
+        '''
+        srv_names = ['com.apple.atru', 'com', 'apple']
+        cmd = 'disabled services = {\n\t"com.saltstack.salt.minion" => false\n\t"com.apple.atrun" => true\n}'
+        for name in srv_names:
+            with patch.object(mac_service, 'launchctl', MagicMock(return_value=cmd)):
+                self.assertFalse(mac_service.disabled(name))
+
+    def test_service_disabled_status_upper_case(self):
+        '''
+        test service.disabled when disabled status is uppercase
+        '''
+        srv_name = 'com.apple.atrun'
+        cmd = 'disabled services = {\n\t"com.saltstack.salt.minion" => false\n\t"com.apple.atrun" => True\n{'
+
+        with patch.object(mac_service, 'launchctl', MagicMock(return_value=cmd)):
+            self.assertTrue(mac_service.disabled(srv_name))

tests/unit/test_fileclient.py

@@ -50,3 +50,14 @@ class FileclientTestCase(TestCase):
                 with self.assertRaises(OSError):
                     with Client(self.opts)._cache_loc('testfile') as c_ref_itr:
                         assert c_ref_itr == '/__test__/files/base/testfile'
+
+    def test_extrn_path_with_long_filename(self):
+        safe_file_name = os.path.split(Client(self.opts)._extrn_path('https://test.com/' + ('A' * 254), 'base'))[-1]
+        assert safe_file_name == 'A' * 254
+
+        oversized_file_name = os.path.split(Client(self.opts)._extrn_path('https://test.com/' + ('A' * 255), 'base'))[-1]
+        assert len(oversized_file_name) < 256
+        assert oversized_file_name != 'A' * 255
+
+        oversized_file_with_query_params = os.path.split(Client(self.opts)._extrn_path('https://test.com/file?' + ('A' * 255), 'base'))[-1]
+        assert len(oversized_file_with_query_params) < 256

tests/unit/test_state.py

@@ -7,6 +7,7 @@
 from __future__ import absolute_import, print_function, unicode_literals
 import copy
 import os
+import shutil
 import tempfile
 
 # Import Salt Testing libs
@@ -18,6 +19,7 @@ from tests.support.mock import (
     MagicMock,
     patch)
 from tests.support.mixins import AdaptedConfigurationTestCaseMixin
+from tests.support.paths import BASE_FILES
 
 # Import Salt libs
 import salt.exceptions
@@ -76,9 +78,9 @@ class StateCompilerTestCase(TestCase, AdaptedConfigurationTestCaseMixin):
 class HighStateTestCase(TestCase, AdaptedConfigurationTestCaseMixin):
     def setUp(self):
         root_dir = tempfile.mkdtemp(dir=integration.TMP)
-        state_tree_dir = os.path.join(root_dir, 'state_tree')
+        self.state_tree_dir = os.path.join(root_dir, 'state_tree')
         cache_dir = os.path.join(root_dir, 'cachedir')
-        for dpath in (root_dir, state_tree_dir, cache_dir):
+        for dpath in (root_dir, self.state_tree_dir, cache_dir):
             if not os.path.isdir(dpath):
                 os.makedirs(dpath)
 
@@ -87,7 +89,7 @@ class HighStateTestCase(TestCase, AdaptedConfigurationTestCaseMixin):
         overrides['state_events'] = False
         overrides['id'] = 'match'
         overrides['file_client'] = 'local'
-        overrides['file_roots'] = dict(base=[state_tree_dir])
+        overrides['file_roots'] = dict(base=[self.state_tree_dir])
         overrides['cachedir'] = cache_dir
         overrides['test'] = False
         self.config = self.get_temp_config('minion', **overrides)
@@ -148,6 +150,28 @@ class HighStateTestCase(TestCase, AdaptedConfigurationTestCaseMixin):
         self.assertEqual(state_usage_dict['base']['used'], ['state.a', 'state.b'])
         self.assertEqual(state_usage_dict['base']['unused'], ['state.c'])
 
+    def test_find_sls_ids_with_exclude(self):
+        '''
+        See https://github.com/saltstack/salt/issues/47182
+        '''
+        sls_dir = 'issue-47182'
+        shutil.copytree(
+            os.path.join(BASE_FILES, sls_dir),
+            os.path.join(self.state_tree_dir, sls_dir)
+        )
+        shutil.move(
+            os.path.join(self.state_tree_dir, sls_dir, 'top.sls'),
+            self.state_tree_dir
+        )
+        # Manually compile the high data. We don't have to worry about all of
+        # the normal error checking we do here since we know that all the SLS
+        # files exist and there is no whitelist/blacklist being used.
+        top = self.highstate.get_top()  # pylint: disable=assignment-from-none
+        matches = self.highstate.top_matches(top)
+        high, _ = self.highstate.render_highstate(matches)
+        ret = salt.state.find_sls_ids('issue-47182.stateA.newer', high)
+        self.assertEqual(ret, [('somestuff', 'cmd')])
+
 
 class TopFileMergeTestCase(TestCase, AdaptedConfigurationTestCaseMixin):
     '''