diff --git a/.kitchen.yml b/.kitchen.yml index ba2266c8b3..cf74731163 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -36,7 +36,7 @@ provisioner: require_chef: false remote_states: name: git://github.com/saltstack/salt-jenkins.git - branch: oxygen + branch: 2018.3 repo: git testingdir: /testing salt_copy_filter: diff --git a/.testing.pylintrc b/.testing.pylintrc index 832ff814e3..ccb20da64d 100644 --- a/.testing.pylintrc +++ b/.testing.pylintrc @@ -29,6 +29,25 @@ load-plugins=saltpylint.pep8, # Don't bump this values on PyLint 1.4.0 - Know bug that ignores the passed --rcfile jobs=1 +# List of blacklisted functions and suggested replacements +# +# NOTE: This pylint check will infer the full name of the function by walking +# back up from the function name to the parent, to the parent's parent, etc., +# and this means that functions which come from platform-specific modules need +# to be referenced using name of the module from which the function was +# imported. This happens a lot in the os and os.path modules. Functions from +# os.path should be defined using posixpath.funcname and ntpath.funcname, while +# functions from os should be defined using posix.funcname and nt.funcname. +# +# When defining a blacklisted function, the format is: +# +# = +# +# The replacement text will be included in the alert message. +# +blacklisted-functions=posix.umask=salt.utils.files.set_umask or get_umask, + nt.umask=salt.utils.files.set_umask or get_umask + # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no diff --git a/doc/_themes/saltstack2/static/images/DOCBANNER.jpg b/doc/_themes/saltstack2/static/images/DOCBANNER.jpg index 886e3a9b7c..f68f0787bb 100644 Binary files a/doc/_themes/saltstack2/static/images/DOCBANNER.jpg and b/doc/_themes/saltstack2/static/images/DOCBANNER.jpg differ diff --git a/doc/conf.py b/doc/conf.py index 17c8d25d4d..0b02718ffa 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -341,10 +341,15 @@ rst_prolog = """\ | md5

-.. |osxdownload| raw:: html +.. |osxdownloadpy2| raw:: html -

x86_64: salt-{release}-x86_64.pkg - | md5

+

x86_64: salt-{release}-py2-x86_64.pkg + | md5

+ +.. |osxdownloadpy3| raw:: html + +

x86_64: salt-{release}-py3-x86_64.pkg + | md5

""".format(release=release) diff --git a/doc/topics/cloud/config.rst b/doc/topics/cloud/config.rst index a8bb1684ef..b721f8421f 100644 --- a/doc/topics/cloud/config.rst +++ b/doc/topics/cloud/config.rst @@ -344,7 +344,35 @@ be set in the configuration file to enable interfacing with GoGrid: OpenStack --------- -.. automodule:: salt.cloud.clouds.openstack +Using Salt for OpenStack uses the `shade ` driver managed by the +openstack-infra team. + +This driver can be configured using the ``/etc/openstack/clouds.yml`` file with +`os-client-config ` + +.. code-block:: yaml + myopenstack: + driver: openstack + region_name: RegionOne + cloud: mycloud + +Or by just configuring the same auth block directly in the cloud provider config. + +.. code-block:: yaml + myopenstack: + driver: openstack + region_name: RegionOne + auth: + username: 'demo' + password: secret + project_name: 'demo' + auth_url: 'http://openstack/identity' + +Both of these methods support using the +`vendor ` +options. + +For more information, look at :mod:`Openstack Cloud Driver Docs ` DigitalOcean ------------ diff --git a/doc/topics/cloud/openstack.rst b/doc/topics/cloud/openstack.rst new file mode 100644 index 0000000000..ccb1fefba8 --- /dev/null +++ b/doc/topics/cloud/openstack.rst @@ -0,0 +1,5 @@ +============================== +Getting Started with Openstack +============================== + +.. automodule:: salt.cloud.clouds.openstack diff --git a/doc/topics/cloud/rackspace.rst b/doc/topics/cloud/rackspace.rst deleted file mode 100644 index 45df0b8553..0000000000 --- a/doc/topics/cloud/rackspace.rst +++ /dev/null @@ -1,188 +0,0 @@ -============================== -Getting Started With Rackspace -============================== - -Rackspace is a major public cloud platform which may be configured using either -the `openstack` driver. - - -Dependencies -============ -* Libcloud >= 0.13.2 - - -Configuration -============= -To use the `openstack` driver (recommended), set up the cloud configuration at - ``/etc/salt/cloud.providers`` or - ``/etc/salt/cloud.providers.d/rackspace.conf``: - -.. code-block:: yaml - - my-rackspace-config: - # Set the location of the salt-master - # - minion: - master: saltmaster.example.com - - # Configure Rackspace using the OpenStack plugin - # - identity_url: 'https://identity.api.rackspacecloud.com/v2.0/tokens' - compute_name: cloudServersOpenStack - protocol: ipv4 - - # Set the compute region: - # - compute_region: DFW - - # Configure Rackspace authentication credentials - # - user: myname - tenant: 123456 - apikey: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - - driver: openstack - - -.. note:: - .. versionchanged:: 2015.8.0 - - The ``provider`` parameter in cloud provider definitions was renamed to ``driver``. This - change was made to avoid confusion with the ``provider`` parameter that is used in cloud profile - definitions. Cloud provider definitions now use ``driver`` to refer to the Salt cloud module that - provides the underlying functionality to connect to a cloud host, while cloud profiles continue - to use ``provider`` to refer to provider configurations that you define. - -Compute Region -============== - -Rackspace currently has six compute regions which may be used: - -.. code-block:: bash - - DFW -> Dallas/Forth Worth - ORD -> Chicago - SYD -> Sydney - LON -> London - IAD -> Northern Virginia - HKG -> Hong Kong - -Note: Currently the LON region is only available with a UK account, and UK accounts cannot access other regions - -Authentication -============== - -The ``user`` is the same user as is used to log into the Rackspace Control -Panel. The ``tenant`` and ``apikey`` can be found in the API Keys area of the -Control Panel. The ``apikey`` will be labeled as API Key (and may need to be -generated), and ``tenant`` will be labeled as Cloud Account Number. - -An initial profile can be configured in ``/etc/salt/cloud.profiles`` or -``/etc/salt/cloud.profiles.d/rackspace.conf``: - -.. code-block:: yaml - - openstack_512: - provider: my-rackspace-config - size: 512 MB Standard - image: Ubuntu 12.04 LTS (Precise Pangolin) - -To instantiate a machine based on this profile: - -.. code-block:: bash - - # salt-cloud -p openstack_512 myinstance - -This will create a virtual machine at Rackspace with the name ``myinstance``. -This operation may take several minutes to complete, depending on the current -load at the Rackspace data center. - -Once the instance has been created with salt-minion installed, connectivity to -it can be verified with Salt: - -.. code-block:: bash - - # salt myinstance test.ping - -RackConnect Environments ------------------------- - -Rackspace offers a hybrid hosting configuration option called RackConnect that -allows you to use a physical firewall appliance with your cloud servers. When -this service is in use the public_ip assigned by nova will be replaced by a NAT -ip on the firewall. For salt-cloud to work properly it must use the newly -assigned "access ip" instead of the Nova assigned public ip. You can enable that -capability by adding this to your profiles: - -.. code-block:: yaml - - openstack_512: - provider: my-openstack-config - size: 512 MB Standard - image: Ubuntu 12.04 LTS (Precise Pangolin) - rackconnect: True - -Managed Cloud Environments --------------------------- - -Rackspace offers a managed service level of hosting. As part of the managed -service level you have the ability to choose from base of lamp installations on -cloud server images. The post build process for both the base and the lamp -installations used Chef to install things such as the cloud monitoring agent and -the cloud backup agent. It also takes care of installing the lamp stack if -selected. In order to prevent the post installation process from stomping over -the bootstrapping you can add the below to your profiles. - -.. code-block:: yaml - - openstack_512: - provider: my-rackspace-config - size: 512 MB Standard - image: Ubuntu 12.04 LTS (Precise Pangolin) - managedcloud: True - -First and Next Generation Images --------------------------------- - -Rackspace provides two sets of virtual machine images, *first*, and *next* -generation. As of ``0.8.9`` salt-cloud will default to using the *next* -generation images. To force the use of first generation images, on the profile -configuration please add: - -.. code-block:: yaml - - FreeBSD-9.0-512: - provider: my-rackspace-config - size: 512 MB Standard - image: FreeBSD 9.0 - force_first_gen: True - -Private Subnets ---------------- -By default salt-cloud will not add Rackspace private networks to new servers. To enable -a private network to a server instantiated by salt cloud, add the following section -to the provider file (typically ``/etc/salt/cloud.providers.d/rackspace.conf``) - -.. code-block:: yaml - - networks: - - fixed: - # This is the private network - - private-network-id - # This is Rackspace's "PublicNet" - - 00000000-0000-0000-0000-000000000000 - # This is Rackspace's "ServiceNet" - - 11111111-1111-1111-1111-111111111111 - -To get the Rackspace private network ID, go to Networking, Networks and hover over the private network name. - -The order of the networks in the above code block does not map to the order of the -ethernet devices on newly created servers. Public IP will always be first ( eth0 ) -followed by servicenet ( eth1 ) and then private networks. - -Enabling the private network per above gives the option of using the private subnet for -all master-minion communication, including the bootstrap install of salt-minion. To -enable the minion to use the private subnet, update the master: line in the minion: -section of the providers file. To configure the master to only listen on the private -subnet IP, update the interface: line in the /etc/salt/master file to be the private -subnet IP of the salt master. diff --git a/doc/topics/installation/osx.rst b/doc/topics/installation/osx.rst index 66ee787ca1..ce78e347d4 100644 --- a/doc/topics/installation/osx.rst +++ b/doc/topics/installation/osx.rst @@ -8,7 +8,8 @@ Installation from the Official SaltStack Repository =================================================== **Latest stable build from the selected branch**: -|osxdownload| +|osxdownloadpy2| +|osxdownloadpy3| The output of ``md5 `` should match the contents of the corresponding md5 file. diff --git a/doc/topics/releases/2018.3.0.rst b/doc/topics/releases/2018.3.0.rst index fb55f3e680..3444a0f226 100644 --- a/doc/topics/releases/2018.3.0.rst +++ b/doc/topics/releases/2018.3.0.rst @@ -665,6 +665,37 @@ The Windows installer will now display command-line help when a help switch Salt Cloud Features ------------------- +OpenStack Revamp +================ + +The OpenStack Driver has been rewritten mostly from scratch. Salt is now using +the `shade driver `. + +With this, the ``nova`` driver is being deprecated. + +:mod:`openstack driver ` + +There have also been several new modules and states added for managing OpenStack +setups using shade as well. + +:mod:`keystone ` +:mod:`keystone role grant ` +:mod:`keystone group ` +:mod:`keystone role ` +:mod:`keystone service ` +:mod:`keystone user ` +:mod:`keystone domain ` +:mod:`keystone project ` +:mod:`keystone endpoint ` +:mod:`glance ` +:mod:`glance_image ` +:mod:`neutron ` +:mod:`neutron subnet ` +:mod:`neutron secgroup ` +:mod:`neutron secgroup rule ` +:mod:`neutron network ` + + Pre-Flight Commands =================== @@ -1557,6 +1588,14 @@ PyCrypto is used as it was in the previous releases. M2Crypto is used in the same way as PyCrypto so there would be no compatibility issues, different nodes could use different backends. +NaCL Module and Runner changes +------------------------------ + +In addition to argument changes in both the NaCL module and runner for future +deprecation in the Fluorine release, the default box_type has changed from +`secretbox` to `sealedbox`. SecretBox is data encrypted using private key +`sk` and Sealedbox is encrypted using public key `pk` + Deprecations ------------ @@ -1617,6 +1656,15 @@ The ``win_service`` module had the following changes: - The ``type`` option was removed from the ``create`` function. Please use ``service_type`` instead. +The ``nacl`` module had the following changes: + +- The ``key_file`` option was replaced in the ``keygen``, ``enc`` and ``dec`` +functions. Please use the ``sk_file`` option instead. + +- The ``key`` option was replaced in the ``keygen``, ``enc`` and ``dec`` +functions. Please use the ``sk`` option instead. + + Runner Deprecations =================== @@ -1625,6 +1673,14 @@ The ``manage`` runner had the following changes: - The ``root_user`` kwarg was removed from the ``bootstrap`` function. Please use ``salt-ssh`` roster entries for the host instead. +The ``nacl`` runner had the following changes: + +- The ``key_file`` option was replaced in the ``keygen``, ``enc`` and ``dec`` +functions. Please use the ``sk_file`` option instead. + +- The ``key`` option was replaced in the ``keygen``, ``enc`` and ``dec`` +functions. Please use the ``sk`` option instead. + State Deprecations ================== diff --git a/doc/topics/releases/fluorine.rst b/doc/topics/releases/fluorine.rst index 575fa6a7ad..7bc49da89d 100644 --- a/doc/topics/releases/fluorine.rst +++ b/doc/topics/releases/fluorine.rst @@ -27,6 +27,27 @@ syndic respects :conf_minion:`enable_legacy_startup_events` as well. Deprecations ------------ +Module Deprecations +=================== + +The ``win_update`` module has been removed. It has been replaced by ``win_wua`` +module. + +The ``win_wua`` module had the following changes: + +- Support for the ``download_update`` function has been removed. Please use the + ``download`` function instead. +- Support for the ``download_updates`` function has been removed. Please use the + ``download`` function instead. +- Support for the ``install_update`` function has been removed. Please use the + ``install`` function instead. +- Support for the ``install_updates`` function has been removed. Please use the + ``install`` function instead. +- Support for the ``list_update`` function has been removed. Please use the + ``get`` function instead. +- Support for the ``list_updates`` function has been removed. Please use the + ``list`` function instead. + Pillar Deprecations =================== @@ -49,6 +70,40 @@ The ``cache`` roster had the following changes: State Deprecations ================== +The ``docker`` state has been removed. The following functions should be used +instead. + +- The ``docker.running`` function was removed. Please update applicable SLS files + to use the ``docker_container.running`` function instead. +- The ``docker.stopped`` function was removed. Please update applicable SLS files + to use the ``docker_container.stopped`` function instead. +- The ``docker.absent`` function was removed. Please update applicable SLS files + to use the ``docker_container.absent`` function instead. +- The ``docker.absent`` function was removed. Please update applicable SLS files + to use the ``docker_container.absent`` function instead. +- The ``docker.network_present`` function was removed. Please update applicable + SLS files to use the ``docker_network.present`` function instead. +- The ``docker.network_absent`` function was removed. Please update applicable + SLS files to use the ``docker_network.absent`` function instead. +- The ``docker.image_present`` function was removed. Please update applicable SLS + files to use the ``docker_image.present`` function instead. +- The ``docker.image_absent`` function was removed. Please update applicable SLS + files to use the ``docker_image.absent`` function instead. +- The ``docker.volume_present`` function was removed. Please update applicable SLS + files to use the ``docker_volume.present`` function instead. +- The ``docker.volume_absent`` function was removed. Please update applicable SLS + files to use the ``docker_volume.absent`` function instead. + +The ``docker_network`` state had the following changes: + +- Support for the ``driver`` option has been removed from the ``absent`` function. + This option had no functionality in ``docker_network.absent``. + +The ``git`` state had the following changes: + +- Support for the ``ref`` option in the ``detached`` state has been removed. + Please use the ``rev`` option instead. + The ``k8s`` state has been removed. The following functions should be used instead: @@ -58,3 +113,5 @@ instead: files to use the ``kubernetes.node_label_present`` function instead. - The ``k8s.label_folder_absent`` function was removed. Please update applicable SLS files to use the ``kubernetes.node_label_folder_absent`` function instead. + +The ``win_update`` state has been removed. Please use the ``win_wua`` state instead. diff --git a/doc/topics/sdb/index.rst b/doc/topics/sdb/index.rst index c4d94120e2..cdc66b21f6 100644 --- a/doc/topics/sdb/index.rst +++ b/doc/topics/sdb/index.rst @@ -79,22 +79,12 @@ from the ``kevinopenstack`` profile above, you would use: salt-call sdb.get sdb://kevinopenstack/password -Some drivers use slightly more complex URIs. For instance, the ``vault`` driver -requires the full path to where the key is stored, followed by a question mark, -followed by the key to be retrieved. If you were using a profile called -``myvault``, you would use a URI that looks like: - -.. code-block:: bash - - salt-call sdb.get 'sdb://myvault/secret/salt?saltstack' - Setting a value uses the same URI as would be used to retrieve it, followed -by the value as another argument. For the above ``myvault`` URI, you would set -a new value using a command like: +by the value as another argument. .. code-block:: bash - salt-call sdb.set 'sdb://myvault/secret/salt?saltstack' 'super awesome' + salt-call sdb.set 'sdb://myvault/secret/salt/saltstack' 'super awesome' Deleting values (if supported by the driver) is done pretty much the same way as getting them. Provided that you have a profile called ``mykvstore`` that uses @@ -109,8 +99,8 @@ the runner system: .. code-block:: bash - salt-run sdb.get 'sdb://myvault/secret/salt?saltstack' - salt-run sdb.set 'sdb://myvault/secret/salt?saltstack' 'super awesome' + salt-run sdb.get 'sdb://myvault/secret/salt/saltstack' + salt-run sdb.set 'sdb://myvault/secret/salt/saltstack' 'super awesome' salt-run sdb.delete 'sdb://mykvstore/foobar' diff --git a/salt/auth/ldap.py b/salt/auth/ldap.py index 84d5f83b1d..cbfb03a2f2 100644 --- a/salt/auth/ldap.py +++ b/salt/auth/ldap.py @@ -361,7 +361,7 @@ def groups(username, **kwargs): [salt.utils.stringutils.to_str(_config('accountattributename')), str('cn')]) # future lint: disable=blacklisted-function for entry, result in search_results: - for user in result[_config('accountattributename'), _config('groupattribute')]: + for user in result[_config('accountattributename')]: if username == salt.utils.stringutils.to_unicode(user).split(',')[0].split('=')[-1]: group_list.append(entry.split(',')[0].split('=')[-1]) diff --git a/salt/cli/api.py b/salt/cli/api.py index 08112cf082..ebb72107f7 100644 --- a/salt/cli/api.py +++ b/salt/cli/api.py @@ -9,11 +9,11 @@ # Import Python libs from __future__ import absolute_import, print_function, unicode_literals -import os import logging # Import Salt libs import salt.client.netapi +import salt.utils.files import salt.utils.parsers as parsers from salt.utils.verify import check_user, verify_files, verify_log @@ -42,9 +42,8 @@ class SaltAPI(parsers.SaltAPIParser): 'udp://', 'file://')): # Logfile is not using Syslog, verify - current_umask = os.umask(0o027) - verify_files([logfile], self.config['user']) - os.umask(current_umask) + with salt.utils.files.set_umask(0o027): + verify_files([logfile], self.config['user']) except OSError as err: log.exception('Failed to prepare salt environment') self.shutdown(err.errno) diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py index ffaf965954..e82b95b0e5 100644 --- a/salt/client/ssh/__init__.py +++ b/salt/client/ssh/__init__.py @@ -1031,6 +1031,7 @@ class Single(object): opts_pkg['pillar_roots'] = self.opts['pillar_roots'] opts_pkg['ext_pillar'] = self.opts['ext_pillar'] opts_pkg['extension_modules'] = self.opts['extension_modules'] + opts_pkg['module_dirs'] = self.opts['module_dirs'] opts_pkg['_ssh_version'] = self.opts['_ssh_version'] opts_pkg['__master_opts__'] = self.context['master_opts'] if '_caller_cachedir' in self.opts: diff --git a/salt/client/ssh/ssh_py_shim.py b/salt/client/ssh/ssh_py_shim.py index e46220fc80..7335c2b0f2 100644 --- a/salt/client/ssh/ssh_py_shim.py +++ b/salt/client/ssh/ssh_py_shim.py @@ -106,9 +106,11 @@ def need_deployment(): ''' if os.path.exists(OPTIONS.saltdir): shutil.rmtree(OPTIONS.saltdir) - old_umask = os.umask(0o077) - os.makedirs(OPTIONS.saltdir) - os.umask(old_umask) + old_umask = os.umask(0o077) # pylint: disable=blacklisted-function + try: + os.makedirs(OPTIONS.saltdir) + finally: + os.umask(old_umask) # pylint: disable=blacklisted-function # Verify perms on saltdir if not is_windows(): euid = os.geteuid() @@ -158,10 +160,10 @@ def unpack_thin(thin_path): Unpack the Salt thin archive. ''' tfile = tarfile.TarFile.gzopen(thin_path) - old_umask = os.umask(0o077) + old_umask = os.umask(0o077) # pylint: disable=blacklisted-function tfile.extractall(path=OPTIONS.saltdir) tfile.close() - os.umask(old_umask) + os.umask(old_umask) # pylint: disable=blacklisted-function try: os.unlink(thin_path) except OSError: @@ -189,10 +191,10 @@ def unpack_ext(ext_path): 'minion', 'extmods') tfile = tarfile.TarFile.gzopen(ext_path) - old_umask = os.umask(0o077) + old_umask = os.umask(0o077) # pylint: disable=blacklisted-function tfile.extractall(path=modcache) tfile.close() - os.umask(old_umask) + os.umask(old_umask) # pylint: disable=blacklisted-function os.unlink(ext_path) ver_path = os.path.join(modcache, 'ext_version') ver_dst = os.path.join(OPTIONS.saltdir, 'ext_version') @@ -299,7 +301,7 @@ def main(argv): # pylint: disable=W0613 sys.stderr.write(OPTIONS.delimiter + '\n') sys.stderr.flush() if OPTIONS.cmd_umask is not None: - old_umask = os.umask(OPTIONS.cmd_umask) + old_umask = os.umask(OPTIONS.cmd_umask) # pylint: disable=blacklisted-function if OPTIONS.tty: # Returns bytes instead of string on python 3 stdout, _ = subprocess.Popen(salt_argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() @@ -313,7 +315,7 @@ def main(argv): # pylint: disable=W0613 else: subprocess.call(salt_argv) if OPTIONS.cmd_umask is not None: - os.umask(old_umask) + os.umask(old_umask) # pylint: disable=blacklisted-function if __name__ == '__main__': sys.exit(main(sys.argv)) diff --git a/salt/client/ssh/wrapper/state.py b/salt/client/ssh/wrapper/state.py index 89b444f139..78d0f3d1ea 100644 --- a/salt/client/ssh/wrapper/state.py +++ b/salt/client/ssh/wrapper/state.py @@ -491,17 +491,18 @@ def request(mods=None, 'kwargs': kwargs } }) - cumask = os.umask(0o77) - try: - if salt.utils.platform.is_windows(): - # Make sure cache file isn't read-only - __salt__['cmd.run']('attrib -R "{0}"'.format(notify_path)) - with salt.utils.files.fopen(notify_path, 'w+b') as fp_: - serial.dump(req, fp_) - except (IOError, OSError): - msg = 'Unable to write state request file {0}. Check permission.' - log.error(msg.format(notify_path)) - os.umask(cumask) + with salt.utils.files.set_umask(0o077): + try: + if salt.utils.platform.is_windows(): + # Make sure cache file isn't read-only + __salt__['cmd.run']('attrib -R "{0}"'.format(notify_path)) + with salt.utils.files.fopen(notify_path, 'w+b') as fp_: + serial.dump(req, fp_) + except (IOError, OSError): + log.error( + 'Unable to write state request file %s. Check permission.', + notify_path + ) return ret @@ -557,17 +558,18 @@ def clear_request(name=None): req.pop(name) else: return False - cumask = os.umask(0o77) - try: - if salt.utils.platform.is_windows(): - # Make sure cache file isn't read-only - __salt__['cmd.run']('attrib -R "{0}"'.format(notify_path)) - with salt.utils.files.fopen(notify_path, 'w+b') as fp_: - serial.dump(req, fp_) - except (IOError, OSError): - msg = 'Unable to write state request file {0}. Check permission.' - log.error(msg.format(notify_path)) - os.umask(cumask) + with salt.utils.files.set_umask(0o077): + try: + if salt.utils.platform.is_windows(): + # Make sure cache file isn't read-only + __salt__['cmd.run']('attrib -R "{0}"'.format(notify_path)) + with salt.utils.files.fopen(notify_path, 'w+b') as fp_: + serial.dump(req, fp_) + except (IOError, OSError): + log.error( + 'Unable to write state request file %s. Check permission.', + notify_path + ) return True diff --git a/salt/cloud/clouds/nova.py b/salt/cloud/clouds/nova.py index d520afbcb7..737abd9f8e 100644 --- a/salt/cloud/clouds/nova.py +++ b/salt/cloud/clouds/nova.py @@ -264,6 +264,12 @@ def __virtual__(): if get_dependencies() is False: return False + __utils__['versions.warn_until']( + 'Neon', + 'This driver has been deprecated and will be removed in the ' + '{version} release of Salt. Please use the openstack driver instead.' + ) + return __virtualname__ diff --git a/salt/cloud/clouds/openstack.py b/salt/cloud/clouds/openstack.py index 62ef280410..5646dc64e2 100644 --- a/salt/cloud/clouds/openstack.py +++ b/salt/cloud/clouds/openstack.py @@ -72,6 +72,7 @@ Or if you need to use a profile to setup some extra stuff, it can be passed as a username: rackusername api_key: myapikey region_name: ORD + auth_type: rackspace_apikey And this will pull in the profile for rackspace and setup all the correct options for the auth_url and different api versions for services. @@ -101,6 +102,23 @@ The salt specific ones are: This is the minimum setup required. +If metadata is set to make sure that the host has finished setting up the +`wait_for_metadata` can be set. + +.. code-block:: yaml + + centos: + provider: myopenstack + image: CentOS 7 + size: ds1G + ssh_key_name: mykey + ssh_key_file: /root/.ssh/id_rsa + meta: + build_config: rack_user_only + wait_for_metadata: + rax_service_level_automation: Complete + rackconnect_automation_status: DEPLOYED + Anything else from the create_server_ docs can be passed through here. - **image**: Image dict, name or ID to boot with. image is required @@ -678,12 +696,18 @@ def create(vm_): data = request_instance(conn=conn, call='action', vm_=vm_) log.debug('VM is now running') - def __query_node_ip(vm_): + def __query_node(vm_): data = show_instance(vm_['name'], conn=conn, call='action') + if 'wait_for_metadata' in vm_: + for key, value in six.iteritems(vm_.get('wait_for_metadata', {})): + log.debug('Waiting for metadata: {0}={1}'.format(key, value)) + if data['metadata'].get(key, None) != value: + log.debug('Metadata is not ready: {0}={1}'.format(key, data['metadata'].get(key, None))) + return False return preferred_ip(vm_, data[ssh_interface(vm_)]) try: - ip_address = __utils__['cloud.wait_for_ip']( - __query_node_ip, + ip_address = __utils__['cloud.wait_for_fun']( + __query_node, update_args=(vm_,) ) except (SaltCloudExecutionTimeout, SaltCloudExecutionFailure) as exc: diff --git a/salt/cloud/clouds/vmware.py b/salt/cloud/clouds/vmware.py index 6696f22d39..c04cf52dc9 100644 --- a/salt/cloud/clouds/vmware.py +++ b/salt/cloud/clouds/vmware.py @@ -2684,14 +2684,15 @@ def create(vm_): non_hostname_chars = compile(r'[^\w-]') if search(non_hostname_chars, vm_name): hostName = split(non_hostname_chars, vm_name, maxsplit=1)[0] + domainName = split(non_hostname_chars, vm_name, maxsplit=1)[-1] else: hostName = vm_name - domainName = hostName.split('.', 1)[-1] + domainName = domain if 'Windows' not in object_ref.config.guestFullName: identity = vim.vm.customization.LinuxPrep() identity.hostName = vim.vm.customization.FixedName(name=hostName) - identity.domain = domainName if hostName != domainName else domain + identity.domain = domainName else: identity = vim.vm.customization.Sysprep() identity.guiUnattended = vim.vm.customization.GuiUnattended() diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 7aba4e0ee6..6cd1ef2ecf 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -653,10 +653,11 @@ VALID_OPTS = { 's3fs_update_interval': int, 'svnfs_update_interval': int, - 'git_pillar_base': six.string_types, - 'git_pillar_branch': six.string_types, - 'git_pillar_env': six.string_types, - 'git_pillar_root': six.string_types, + # NOTE: git_pillar_base, git_pillar_branch, git_pillar_env, and + # git_pillar_root omitted here because their values could conceivably be + # loaded as non-string types, which is OK because git_pillar will normalize + # them to strings. But rather than include all the possible types they + # could be, we'll just skip type-checking. 'git_pillar_ssl_verify': bool, 'git_pillar_global_lock': bool, 'git_pillar_user': six.string_types, @@ -668,12 +669,11 @@ VALID_OPTS = { 'git_pillar_refspecs': list, 'git_pillar_includes': bool, 'git_pillar_verify_config': bool, + # NOTE: gitfs_base, gitfs_mountpoint, and gitfs_root omitted here because + # their values could conceivably be loaded as non-string types, which is OK + # because gitfs will normalize them to strings. But rather than include all + # the possible types they could be, we'll just skip type-checking. 'gitfs_remotes': list, - 'gitfs_mountpoint': six.string_types, - 'gitfs_root': six.string_types, - 'gitfs_base': six.string_types, - 'gitfs_user': six.string_types, - 'gitfs_password': six.string_types, 'gitfs_insecure_auth': bool, 'gitfs_privkey': six.string_types, 'gitfs_pubkey': six.string_types, @@ -888,11 +888,14 @@ VALID_OPTS = { 'winrepo_dir': six.string_types, 'winrepo_dir_ng': six.string_types, 'winrepo_cachefile': six.string_types, + # NOTE: winrepo_branch omitted here because its value could conceivably be + # loaded as a non-string type, which is OK because winrepo will normalize + # them to strings. But rather than include all the possible types it could + # be, we'll just skip type-checking. 'winrepo_cache_expire_max': int, 'winrepo_cache_expire_min': int, 'winrepo_remotes': list, 'winrepo_remotes_ng': list, - 'winrepo_branch': six.string_types, 'winrepo_ssl_verify': bool, 'winrepo_user': six.string_types, 'winrepo_password': six.string_types, @@ -1639,6 +1642,7 @@ DEFAULT_MASTER_OPTS = { 'eauth_acl_module': '', 'eauth_tokens': 'localfs', 'extension_modules': os.path.join(salt.syspaths.CACHE_DIR, 'master', 'extmods'), + 'module_dirs': [], 'file_recv': False, 'file_recv_max_size': 100, 'file_buffer_size': 1048576, diff --git a/salt/crypt.py b/salt/crypt.py index bd82d71ff5..f34cf1d660 100644 --- a/salt/crypt.py +++ b/salt/crypt.py @@ -84,8 +84,7 @@ def dropfile(cachedir, user=None): ''' dfn = os.path.join(cachedir, '.dfn') # set a mask (to avoid a race condition on file creation) and store original. - mask = os.umask(191) - try: + with salt.utils.files.set_umask(0o277): log.info('Rotating AES key') if os.path.isfile(dfn): log.info('AES key rotation already requested') @@ -103,8 +102,6 @@ def dropfile(cachedir, user=None): os.chown(dfn, uid, -1) except (KeyError, ImportError, OSError, IOError): pass - finally: - os.umask(mask) # restore original umask def gen_keys(keydir, keyname, keysize, user=None, passphrase=None): @@ -138,17 +135,19 @@ def gen_keys(keydir, keyname, keysize, user=None, passphrase=None): if not os.access(keydir, os.W_OK): raise IOError('Write access denied to "{0}" for user "{1}".'.format(os.path.abspath(keydir), getpass.getuser())) - cumask = os.umask(0o277) - if HAS_M2: - # if passphrase is empty or None use no cipher - if not passphrase: - gen.save_pem(priv, cipher=None) + with salt.utils.files.set_umask(0o277): + if HAS_M2: + # if passphrase is empty or None use no cipher + if not passphrase: + gen.save_pem(priv, cipher=None) + else: + gen.save_pem( + priv, + cipher='des_ede3_cbc', + callback=lambda x: salt.utils.stringutils.to_bytes(passphrase)) else: - gen.save_pem(priv, cipher='des_ede3_cbc', callback=lambda x: six.b(passphrase)) - else: - with salt.utils.files.fopen(priv, 'wb+') as f: - f.write(gen.exportKey('PEM', passphrase)) - os.umask(cumask) + with salt.utils.files.fopen(priv, 'wb+') as f: + f.write(gen.exportKey('PEM', passphrase)) if HAS_M2: gen.save_pub_key(pub) else: diff --git a/salt/daemons/masterapi.py b/salt/daemons/masterapi.py index 48649872db..84537fab3b 100644 --- a/salt/daemons/masterapi.py +++ b/salt/daemons/masterapi.py @@ -202,10 +202,9 @@ def mk_key(opts, user): os.unlink(keyfile) key = salt.crypt.Crypticle.generate_key_string() - cumask = os.umask(191) - with salt.utils.files.fopen(keyfile, 'w+') as fp_: - fp_.write(salt.utils.stringutils.to_str(key)) - os.umask(cumask) + with salt.utils.files.set_umask(0o277): + with salt.utils.files.fopen(keyfile, 'w+') as fp_: + fp_.write(salt.utils.stringutils.to_str(key)) # 600 octal: Read and write access to the owner only. # Write access is necessary since on subsequent runs, if the file # exists, it needs to be written to again. Windows enforces this. diff --git a/salt/fileclient.py b/salt/fileclient.py index 88bd671a1f..76719f6c99 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -143,22 +143,20 @@ class Client(object): saltenv, path) destdir = os.path.dirname(dest) - cumask = os.umask(63) + with salt.utils.files.set_umask(0o077): + # remove destdir if it is a regular file to avoid an OSError when + # running os.makedirs below + if os.path.isfile(destdir): + os.remove(destdir) - # remove destdir if it is a regular file to avoid an OSError when - # running os.makedirs below - if os.path.isfile(destdir): - os.remove(destdir) + # ensure destdir exists + try: + os.makedirs(destdir) + except OSError as exc: + if exc.errno != errno.EEXIST: # ignore if it was there already + raise - # ensure destdir exists - try: - os.makedirs(destdir) - except OSError as exc: - if exc.errno != errno.EEXIST: # ignore if it was there already - raise - - yield dest - os.umask(cumask) + yield dest def get_cachedir(self, cachedir=None): if cachedir is None: diff --git a/salt/grains/core.py b/salt/grains/core.py index 3d6b15e914..b6f6b28aea 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -1927,7 +1927,7 @@ def fqdns(): fqdns.add(socket.gethostbyaddr(ip)[0]) except (socket.error, socket.herror, socket.gaierror, socket.timeout) as e: - log.error("Exception during resolving address: " + str(e)) + log.info("Exception during resolving address: " + str(e)) grains['fqdns'] = sorted(list(fqdns)) return grains diff --git a/salt/key.py b/salt/key.py index 3b936d2e42..80dc4d9c7c 100644 --- a/salt/key.py +++ b/salt/key.py @@ -1044,7 +1044,7 @@ class RaetKey(Key): ''' Use libnacl to generate and safely save a private key ''' - import libnacl.dual # pylint: disable=3rd-party-module-not-gated + import libnacl.dual # pylint: disable=import-error,3rd-party-module-not-gated d_key = libnacl.dual.DualSecret() keydir, keyname, _, _ = self._get_key_attrs(keydir, keyname, keysize, user) @@ -1440,14 +1440,13 @@ class RaetKey(Key): keydata = {'priv': priv, 'sign': sign} path = os.path.join(self.opts['pki_dir'], 'local.key') - c_umask = os.umask(191) - if os.path.exists(path): - #mode = os.stat(path).st_mode - os.chmod(path, stat.S_IWUSR | stat.S_IRUSR) - with salt.utils.files.fopen(path, 'w+b') as fp_: - fp_.write(self.serial.dumps(keydata)) - os.chmod(path, stat.S_IRUSR) - os.umask(c_umask) + with salt.utils.files.set_umask(0o277): + if os.path.exists(path): + #mode = os.stat(path).st_mode + os.chmod(path, stat.S_IWUSR | stat.S_IRUSR) + with salt.utils.files.fopen(path, 'w+') as fp_: + fp_.write(self.serial.dumps(keydata)) + os.chmod(path, stat.S_IRUSR) def delete_local(self): ''' diff --git a/salt/loader.py b/salt/loader.py index d343ebd7a9..fd547b633a 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -22,6 +22,7 @@ from zipimport import zipimporter import salt.config import salt.syspaths import salt.utils.context +import salt.utils.data import salt.utils.dictupdate import salt.utils.event import salt.utils.files @@ -651,7 +652,7 @@ def _load_cached_grains(opts, cfn): try: serial = salt.payload.Serial(opts) with salt.utils.files.fopen(cfn, 'rb') as fp_: - cached_grains = serial.load(fp_) + cached_grains = salt.utils.data.decode(serial.load(fp_)) if not cached_grains: log.debug('Cached grains are empty, cache might be corrupted. Refreshing.') return None @@ -791,35 +792,34 @@ def grains(opts, force_refresh=False, proxy=None): grains_data.update(opts['grains']) # Write cache if enabled if opts.get('grains_cache', False): - cumask = os.umask(0o77) - try: - if salt.utils.platform.is_windows(): - # Late import - import salt.modules.cmdmod - # Make sure cache file isn't read-only - salt.modules.cmdmod._run_quiet('attrib -R "{0}"'.format(cfn)) - with salt.utils.files.fopen(cfn, 'w+b') as fp_: - try: - serial = salt.payload.Serial(opts) - serial.dump(grains_data, fp_) - except TypeError as e: - log.error('Failed to serialize grains cache: %s', e) - raise # re-throw for cleanup - except Exception as e: - log.error('Unable to write to grains cache file %s: %s', cfn, e) - # Based on the original exception, the file may or may not have been - # created. If it was, we will remove it now, as the exception means - # the serialized data is not to be trusted, no matter what the - # exception is. - if os.path.isfile(cfn): - os.unlink(cfn) - os.umask(cumask) + with salt.utils.files.set_umask(0o077): + try: + if salt.utils.platform.is_windows(): + # Late import + import salt.modules.cmdmod + # Make sure cache file isn't read-only + salt.modules.cmdmod._run_quiet('attrib -R "{0}"'.format(cfn)) + with salt.utils.files.fopen(cfn, 'w+b') as fp_: + try: + serial = salt.payload.Serial(opts) + serial.dump(grains_data, fp_) + except TypeError as e: + log.error('Failed to serialize grains cache: %s', e) + raise # re-throw for cleanup + except Exception as e: + log.error('Unable to write to grains cache file %s: %s', cfn, e) + # Based on the original exception, the file may or may not have been + # created. If it was, we will remove it now, as the exception means + # the serialized data is not to be trusted, no matter what the + # exception is. + if os.path.isfile(cfn): + os.unlink(cfn) if grains_deep_merge: salt.utils.dictupdate.update(grains_data, opts['grains']) else: grains_data.update(opts['grains']) - return grains_data + return salt.utils.data.decode(grains_data) # TODO: get rid of? Does anyone use this? You should use raw() instead diff --git a/salt/master.py b/salt/master.py index 83e7bb145a..c9f0bac35c 100644 --- a/salt/master.py +++ b/salt/master.py @@ -578,9 +578,8 @@ class Master(SMaster): # Check to see if we need to create a pillar cache dir if self.opts['pillar_cache'] and not os.path.isdir(os.path.join(self.opts['cachedir'], 'pillar_cache')): try: - prev_umask = os.umask(0o077) - os.mkdir(os.path.join(self.opts['cachedir'], 'pillar_cache')) - os.umask(prev_umask) + with salt.utils.files.set_umask(0o077): + os.mkdir(os.path.join(self.opts['cachedir'], 'pillar_cache')) except OSError: pass diff --git a/salt/modules/archive.py b/salt/modules/archive.py index f9750b938d..48f0efa18e 100644 --- a/salt/modules/archive.py +++ b/salt/modules/archive.py @@ -1077,8 +1077,7 @@ def unzip(zip_file, if not salt.utils.platform.is_windows(): perm = zfile.getinfo(target).external_attr >> 16 if perm == 0: - umask_ = os.umask(0) - os.umask(umask_) + umask_ = salt.utils.files.get_umask() if target.endswith('/'): perm = 0o777 & ~umask_ else: diff --git a/salt/modules/boto_ec2.py b/salt/modules/boto_ec2.py index 93911fc307..11f85383b1 100644 --- a/salt/modules/boto_ec2.py +++ b/salt/modules/boto_ec2.py @@ -657,40 +657,40 @@ def find_images(ami_name=None, executable_by=None, owners=None, image_ids=None, salt myminion boto_ec2.find_images tags='{"mytag": "value"}' ''' + retries = 30 conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - - try: - filter_parameters = {'filters': {}} - - if image_ids: - filter_parameters['image_ids'] = [image_ids] - - if executable_by: - filter_parameters['executable_by'] = [executable_by] - - if owners: - filter_parameters['owners'] = [owners] - - if ami_name: - filter_parameters['filters']['name'] = ami_name - - if tags: - for tag_name, tag_value in six.iteritems(tags): - filter_parameters['filters']['tag:{0}'.format(tag_name)] = tag_value - - images = conn.get_all_images(**filter_parameters) - log.debug('The filters criteria %s matched the following ' - 'images:%s', filter_parameters, images) - - if images: - if return_objs: - return images - return [image.id for image in images] - else: + while retries: + try: + filter_parameters = {'filters': {}} + if image_ids: + filter_parameters['image_ids'] = [image_ids] + if executable_by: + filter_parameters['executable_by'] = [executable_by] + if owners: + filter_parameters['owners'] = [owners] + if ami_name: + filter_parameters['filters']['name'] = ami_name + if tags: + for tag_name, tag_value in six.iteritems(tags): + filter_parameters['filters']['tag:{0}'.format(tag_name)] = tag_value + images = conn.get_all_images(**filter_parameters) + log.debug('The filters criteria %s matched the following ' + 'images:%s', filter_parameters, images) + if images: + if return_objs: + return images + return [image.id for image in images] + else: + return False + except boto.exception.BotoServerError as exc: + if exc.error_code == 'Throttling': + log.debug("Throttled by AWS API, will retry in 5 seconds...") + time.sleep(5) + retries -= 1 + continue + log.error('Failed to convert AMI name `%s` to an AMI ID: %s', ami_name, exc) return False - except boto.exception.BotoServerError as exc: - log.error(exc) - return False + return False def terminate(instance_id=None, name=None, region=None, diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index a56a99100b..25051689af 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -12,6 +12,7 @@ import functools import glob import logging import os +import platform import shutil import subprocess import sys @@ -3344,9 +3345,14 @@ def powershell(cmd, python_shell = True # Append PowerShell Object formatting - cmd += ' | ConvertTo-JSON' - if depth is not None: - cmd += ' -Depth {0}'.format(depth) + # ConvertTo-JSON is only available on Versions of Windows greater than + # `7.1.7600`. We have to use `platform.version` instead of `__grains__` here + # because this function is called by `salt/grains/core.py` before + # `__grains__` is populated + if salt.utils.versions.version_cmp(platform.version(), '7.1.7600') == 1: + cmd += ' | ConvertTo-JSON' + if depth is not None: + cmd += ' -Depth {0}'.format(depth) if encode_cmd: # Convert the cmd to UTF-16LE without a BOM and base64 encode. @@ -3363,7 +3369,7 @@ def powershell(cmd, # caught in a try/catch block. For example, the `Get-WmiObject` command will # often return a "Non Terminating Error". To fix this, make sure # `-ErrorAction Stop` is set in the powershell command - cmd = 'try {' + cmd + '} catch { "{}" | ConvertTo-JSON}' + cmd = 'try {' + cmd + '} catch { "{}" }' # Retrieve the response, while overriding shell with 'powershell' response = run(cmd, diff --git a/salt/modules/file.py b/salt/modules/file.py index 0dcf330171..98a0fc1501 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -5414,30 +5414,25 @@ def manage_file(name, # Create the file, user rw-only if mode will be set to prevent # a small security race problem before the permissions are set - if mode: - current_umask = os.umask(0o77) - - # Create a new file when test is False and source is None - if contents is None: - if not __opts__['test']: - if touch(name): - ret['changes']['new'] = 'file {0} created'.format(name) - ret['comment'] = 'Empty file' - else: - return _error( - ret, 'Empty file {0} not created'.format(name) - ) - else: - if not __opts__['test']: - if touch(name): - ret['changes']['diff'] = 'New file' - else: - return _error( - ret, 'File {0} not created'.format(name) - ) - - if mode: - os.umask(current_umask) + with salt.utils.files.set_umask(0o077 if mode else None): + # Create a new file when test is False and source is None + if contents is None: + if not __opts__['test']: + if touch(name): + ret['changes']['new'] = 'file {0} created'.format(name) + ret['comment'] = 'Empty file' + else: + return _error( + ret, 'Empty file {0} not created'.format(name) + ) + else: + if not __opts__['test']: + if touch(name): + ret['changes']['diff'] = 'New file' + else: + return _error( + ret, 'File {0} not created'.format(name) + ) if contents is not None: # Write the static contents to a temporary file @@ -5471,8 +5466,7 @@ def manage_file(name, # out what mode to use for the new file. if mode is None and not salt.utils.platform.is_windows(): # Get current umask - mask = os.umask(0) - os.umask(mask) + mask = salt.utils.files.get_umask() # Calculate the mode value that results from the umask mode = oct((0o777 ^ mask) & 0o666) diff --git a/salt/modules/glance.py b/salt/modules/glance.py index 2a7caac7e1..16b90c9072 100644 --- a/salt/modules/glance.py +++ b/salt/modules/glance.py @@ -103,6 +103,13 @@ def _auth(profile=None, api_version=2, **connection_args): Only intended to be used within glance-enabled modules ''' + __utils__['versions.warn_until']( + 'Neon', + ( + 'The glance module has been deprecated and will be removed in {version}. ' + 'Please update to using the glanceng module' + ), + ) if profile: prefix = profile + ":keystone." diff --git a/salt/modules/inspectlib/collector.py b/salt/modules/inspectlib/collector.py index 74951391cb..8d1cf3c9a0 100644 --- a/salt/modules/inspectlib/collector.py +++ b/salt/modules/inspectlib/collector.py @@ -513,7 +513,7 @@ if __name__ == '__main__': sys.exit(1) os.setsid() - os.umask(0) + os.umask(0o000) # pylint: disable=blacklisted-function try: pid = os.fork() diff --git a/salt/modules/keystone.py b/salt/modules/keystone.py index 2f62d262f4..ba871f1deb 100644 --- a/salt/modules/keystone.py +++ b/salt/modules/keystone.py @@ -163,6 +163,13 @@ def auth(profile=None, **connection_args): salt '*' keystone.auth ''' + __utils__['versions.warn_until']( + 'Neon', + ( + 'The keystone module has been deprecated and will be removed in {version}. ' + 'Please update to using the keystoneng module', + ), + ) kwargs = _get_kwargs(profile=profile, **connection_args) disc = discover.Discover(auth_url=kwargs['auth_url']) diff --git a/salt/modules/kubernetes.py b/salt/modules/kubernetes.py index 15ab2da8da..4dcc99b658 100644 --- a/salt/modules/kubernetes.py +++ b/salt/modules/kubernetes.py @@ -60,7 +60,9 @@ from salt.exceptions import CommandExecutionError from salt.ext.six import iteritems from salt.ext import six import salt.utils.files +import salt.utils.platform import salt.utils.templates +import salt.utils.versions import salt.utils.yaml from salt.exceptions import TimeoutError from salt.ext.six.moves import range # pylint: disable=import-error @@ -223,11 +225,11 @@ def _cleanup_old(**kwargs): cert = kubernetes.client.configuration.cert_file key = kubernetes.client.configuration.key_file if cert and os.path.exists(cert) and os.path.basename(cert).startswith('salt-kube-'): - salt.utils.safe_rm(cert) + salt.utils.files.safe_rm(cert) if key and os.path.exists(key) and os.path.basename(key).startswith('salt-kube-'): - salt.utils.safe_rm(key) + salt.utils.files.safe_rm(key) if ca and os.path.exists(ca) and os.path.basename(ca).startswith('salt-kube-'): - salt.utils.safe_rm(ca) + salt.utils.files.safe_rm(ca) except Exception: pass diff --git a/salt/modules/nacl.py b/salt/modules/nacl.py index 96a1afb5c0..1f8e27ff4e 100644 --- a/salt/modules/nacl.py +++ b/salt/modules/nacl.py @@ -164,7 +164,6 @@ import salt.utils.stringutils import salt.utils.win_functions import salt.utils.win_dacl - REQ_ERROR = None try: import libnacl.secret @@ -186,9 +185,9 @@ def _get_config(**kwargs): config = { 'box_type': 'sealedbox', 'sk': None, - 'sk_file': '/etc/salt/pki/master/nacl', + 'sk_file': os.path.join(__opts__['pki_dir'], 'master/nacl'), 'pk': None, - 'pk_file': '/etc/salt/pki/master/nacl.pub', + 'pk_file': os.path.join(__opts__['pki_dir'], 'master/nacl.pub'), } config_key = '{0}.config'.format(__virtualname__) try: @@ -233,7 +232,7 @@ def _get_pk(**kwargs): return base64.b64decode(pubkey) -def keygen(sk_file=None, pk_file=None): +def keygen(sk_file=None, pk_file=None, **kwargs): ''' Use libnacl to generate a keypair. @@ -253,6 +252,14 @@ def keygen(sk_file=None, pk_file=None): salt-call nacl.keygen sk_file=/etc/salt/pki/master/nacl pk_file=/etc/salt/pki/master/nacl.pub salt-call --local nacl.keygen ''' + if 'keyfile' in kwargs: + salt.utils.versions.warn_until( + 'Fluorine', + 'The \'keyfile\' argument has been deprecated and will be removed in Salt ' + '{version}. Please use \'sk_file\' argument instead.' + ) + sk_file = kwargs['keyfile'] + if sk_file is None: kp = libnacl.public.SecretKey() return {'sk': base64.b64encode(kp.sk), 'pk': base64.b64encode(kp.pk)} @@ -313,6 +320,25 @@ def enc(data, **kwargs): box_type: secretbox, sealedbox(default) ''' + if 'keyfile' in kwargs: + salt.utils.versions.warn_until( + 'Fluorine', + 'The \'keyfile\' argument has been deprecated and will be removed in Salt ' + '{version}. Please use \'sk_file\' argument instead.' + ) + kwargs['sk_file'] = kwargs['keyfile'] + + if 'key' in kwargs: + salt.utils.versions.warn_until( + 'Fluorine', + 'The \'key\' argument has been deprecated and will be removed in Salt ' + '{version}. Please use \'sk\' argument instead.' + ) + kwargs['sk'] = kwargs['key'] + + # ensure data is in bytes + data = salt.utils.stringutils.to_bytes(data) + box_type = _get_config(**kwargs)['box_type'] if box_type == 'sealedbox': return sealedbox_encrypt(data, **kwargs) @@ -360,6 +386,31 @@ def dec(data, **kwargs): box_type: secretbox, sealedbox(default) ''' + if 'keyfile' in kwargs: + salt.utils.versions.warn_until( + 'Fluorine', + 'The \'keyfile\' argument has been deprecated and will be removed in Salt ' + '{version}. Please use \'sk_file\' argument instead.' + ) + kwargs['sk_file'] = kwargs['keyfile'] + + # set boxtype to `secretbox` to maintain backward compatibility + kwargs['box_type'] = 'secretbox' + + if 'key' in kwargs: + salt.utils.versions.warn_until( + 'Fluorine', + 'The \'key\' argument has been deprecated and will be removed in Salt ' + '{version}. Please use \'sk\' argument instead.' + ) + kwargs['sk'] = kwargs['key'] + + # set boxtype to `secretbox` to maintain backward compatibility + kwargs['box_type'] = 'secretbox' + + # ensure data is in bytes + data = salt.utils.stringutils.to_bytes(data) + box_type = _get_config(**kwargs)['box_type'] if box_type == 'sealedbox': return sealedbox_decrypt(data, **kwargs) @@ -414,6 +465,9 @@ def sealedbox_encrypt(data, **kwargs): salt-call --local nacl.sealedbox_encrypt datatoenc pk_file=/etc/salt/pki/master/nacl.pub salt-call --local nacl.sealedbox_encrypt datatoenc pk='vrwQF7cNiNAVQVAiS3bvcbJUnF0cN6fU9YTZD9mBfzQ=' ''' + # ensure data is in bytes + data = salt.utils.stringutils.to_bytes(data) + pk = _get_pk(**kwargs) b = libnacl.sealed.SealedBox(pk) return base64.b64encode(b.encrypt(data)) @@ -433,6 +487,10 @@ def sealedbox_decrypt(data, **kwargs): ''' if data is None: return None + + # ensure data is in bytes + data = salt.utils.stringutils.to_bytes(data) + sk = _get_sk(**kwargs) keypair = libnacl.public.SecretKey(sk) b = libnacl.sealed.SealedBox(keypair) @@ -452,6 +510,9 @@ def secretbox_encrypt(data, **kwargs): salt-call --local nacl.secretbox_encrypt datatoenc sk_file=/etc/salt/pki/master/nacl salt-call --local nacl.secretbox_encrypt datatoenc sk='YmFkcGFzcwo=' ''' + # ensure data is in bytes + data = salt.utils.stringutils.to_bytes(data) + sk = _get_sk(**kwargs) b = libnacl.secret.SecretBox(sk) return base64.b64encode(b.encrypt(data)) @@ -472,6 +533,10 @@ def secretbox_decrypt(data, **kwargs): ''' if data is None: return None + + # ensure data is in bytes + data = salt.utils.stringutils.to_bytes(data) + key = _get_sk(**kwargs) b = libnacl.secret.SecretBox(key=key) return b.decrypt(base64.b64decode(data)) diff --git a/salt/modules/neutron.py b/salt/modules/neutron.py index 87ca1de7f3..03a5db4699 100644 --- a/salt/modules/neutron.py +++ b/salt/modules/neutron.py @@ -1618,167 +1618,3 @@ def list_agents(profile=None): ''' conn = _auth(profile) return conn.list_agents() - - -# The following is a list of functions that need to be incorporated in the -# neutron module. This list should be updated as functions are added. -# -# update_ipsec_site_connection -# Updates an IPsecSiteConnection. -# update_ikepolicy Updates an IKEPolicy -# update_ipsecpolicy Updates an IPsecPolicy -# list_vips Fetches a list of all load balancer vips for a tenant. -# show_vip Fetches information of a certain load balancer vip. -# create_vip Creates a new load balancer vip. -# update_vip Updates a load balancer vip. -# delete_vip Deletes the specified load balancer vip. -# list_pools Fetches a list of all load balancer pools for a tenant. -# show_pool Fetches information of a certain load balancer pool. -# create_pool Creates a new load balancer pool. -# update_pool Updates a load balancer pool. -# delete_pool Deletes the specified load balancer pool. -# retrieve_pool_stats Retrieves stats for a certain load balancer pool. -# list_members Fetches a list of all load balancer members for -# a tenant. -# show_member Fetches information of a certain load balancer member. -# create_member Creates a new load balancer member. -# update_member Updates a load balancer member. -# delete_member Deletes the specified load balancer member. -# list_health_monitors Fetches a list of all load balancer health monitors for -# a tenant. -# show_health_monitor Fetches information of a certain load balancer -# health monitor. -# create_health_monitor -# Creates a new load balancer health monitor. -# update_health_monitor -# Updates a load balancer health monitor. -# delete_health_monitor -# Deletes the specified load balancer health monitor. -# associate_health_monitor -# Associate specified load balancer health monitor -# and pool. -# disassociate_health_monitor -# Disassociate specified load balancer health monitor -# and pool. -# create_qos_queue Creates a new queue. -# list_qos_queues Fetches a list of all queues for a tenant. -# show_qos_queue Fetches information of a certain queue. -# delete_qos_queue Deletes the specified queue. -# list_agents Fetches agents. -# show_agent Fetches information of a certain agent. -# update_agent Updates an agent. -# delete_agent Deletes the specified agent. -# list_network_gateways -# Retrieve network gateways. -# show_network_gateway Fetch a network gateway. -# create_network_gateway -# Create a new network gateway. -# update_network_gateway -# Update a network gateway. -# delete_network_gateway -# Delete the specified network gateway. -# connect_network_gateway -# Connect a network gateway to the specified network. -# disconnect_network_gateway -# Disconnect a network from the specified gateway. -# list_gateway_devices Retrieve gateway devices. -# show_gateway_device Fetch a gateway device. -# create_gateway_device -# Create a new gateway device. -# update_gateway_device -# Updates a new gateway device. -# delete_gateway_device -# Delete the specified gateway device. -# list_dhcp_agent_hosting_networks -# Fetches a list of dhcp agents hosting a network. -# list_networks_on_dhcp_agent -# Fetches a list of dhcp agents hosting a network. -# add_network_to_dhcp_agent -# Adds a network to dhcp agent. -# remove_network_from_dhcp_agent -# Remove a network from dhcp agent. -# list_l3_agent_hosting_routers -# Fetches a list of L3 agents hosting a router. -# list_routers_on_l3_agent -# Fetches a list of L3 agents hosting a router. -# add_router_to_l3_agent -# Adds a router to L3 agent. -# list_firewall_rules Fetches a list of all firewall rules for a tenant. -# show_firewall_rule Fetches information of a certain firewall rule. -# create_firewall_rule Creates a new firewall rule. -# update_firewall_rule Updates a firewall rule. -# delete_firewall_rule Deletes the specified firewall rule. -# list_firewall_policies -# Fetches a list of all firewall policies for a tenant. -# show_firewall_policy Fetches information of a certain firewall policy. -# create_firewall_policy -# Creates a new firewall policy. -# update_firewall_policy -# Updates a firewall policy. -# delete_firewall_policy -# Deletes the specified firewall policy. -# firewall_policy_insert_rule -# Inserts specified rule into firewall policy. -# firewall_policy_remove_rule -# Removes specified rule from firewall policy. -# list_firewalls Fetches a list of all firewals for a tenant. -# show_firewall Fetches information of a certain firewall. -# create_firewall Creates a new firewall. -# update_firewall Updates a firewall. -# delete_firewall Deletes the specified firewall. -# remove_router_from_l3_agent -# Remove a router from l3 agent. -# get_lbaas_agent_hosting_pool -# Fetches a loadbalancer agent hosting a pool. -# list_pools_on_lbaas_agent -# Fetches a list of pools hosted by -# the loadbalancer agent. -# list_service_providers -# Fetches service providers. -# list_credentials Fetch a list of all credentials for a tenant. -# show_credential Fetch a credential. -# create_credential Create a new credential. -# update_credential Update a credential. -# delete_credential Delete the specified credential. -# list_network_profile_bindings -# Fetch a list of all tenants associated for -# a network profile. -# list_network_profiles -# Fetch a list of all network profiles for a tenant. -# show_network_profile Fetch a network profile. -# create_network_profile -# Create a network profile. -# update_network_profile -# Update a network profile. -# delete_network_profile -# Delete the network profile. -# list_policy_profile_bindings -# Fetch a list of all tenants associated for -# a policy profile. -# list_policy_profiles Fetch a list of all network profiles for a tenant. -# show_policy_profile Fetch a network profile. -# update_policy_profile -# Update a policy profile. -# create_metering_label -# Creates a metering label. -# delete_metering_label -# Deletes the specified metering label. -# list_metering_labels Fetches a list of all metering labels for a tenant. -# show_metering_label Fetches information of a certain metering label. -# create_metering_label_rule -# Creates a metering label rule. -# delete_metering_label_rule -# Deletes the specified metering label rule. -# list_metering_label_rules -# Fetches a list of all metering label rules for a label. -# show_metering_label_rule -# Fetches information of a certain metering label rule. -# list_net_partitions Fetch a list of all network partitions for a tenant. -# show_net_partition etch a network partition. -# create_net_partition Create a network partition. -# delete_net_partition Delete the network partition. -# create_packet_filter Create a new packet filter. -# update_packet_filter Update a packet filter. -# list_packet_filters Fetch a list of all packet filters for a tenant. -# show_packet_filter Fetch information of a certain packet filter. -# delete_packet_filter Delete the specified packet filter. diff --git a/salt/modules/pillar.py b/salt/modules/pillar.py index 5e7a184fba..d320e46754 100644 --- a/salt/modules/pillar.py +++ b/salt/modules/pillar.py @@ -42,8 +42,8 @@ def get(key, Attempt to retrieve the named value from pillar, if the named value is not available return the passed default. The default return is an empty string - except __opts__['pillar_raise_on_missing'] is set to True, in which case a - KeyError will be raised. + except ``__opts__['pillar_raise_on_missing']`` is set to True, in which + case a ``KeyError`` exception will be raised. If the merge parameter is set to ``True``, the default will be recursively merged into the returned pillar data. @@ -53,11 +53,18 @@ def get(key, {'pkg': {'apache': 'httpd'}} - To retrieve the value associated with the apache key in the pkg dict this - key can be passed:: + To retrieve the value associated with the ``apache`` key in the ``pkg`` + dict this key can be passed as:: pkg:apache + key + The pillar key to get value from + + default + If specified, return this value in case when named pillar value does + not exist. + merge : ``False`` If ``True``, the retrieved values will be merged into the passed default. When the default and the retrieved value are both diff --git a/salt/modules/pip.py b/salt/modules/pip.py index f9830056b9..b560612b9e 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -338,6 +338,22 @@ def _process_requirements(requirements, cmd, cwd, saltenv, user): return cleanup_requirements, None +def _format_env_vars(env_vars): + ret = {} + if env_vars: + if isinstance(env_vars, dict): + for key, val in six.iteritems(env_vars): + if not isinstance(key, six.string_types): + key = str(key) # future lint: disable=blacklisted-function + if not isinstance(val, six.string_types): + val = str(val) # future lint: disable=blacklisted-function + ret[key] = val + else: + raise CommandExecutionError( + 'env_vars {0} is not a dictionary'.format(env_vars)) + return ret + + def install(pkgs=None, # pylint: disable=R0912,R0913,R0914 requirements=None, bin_env=None, @@ -811,16 +827,7 @@ def install(pkgs=None, # pylint: disable=R0912,R0913,R0914 cmd_kwargs = dict(saltenv=saltenv, use_vt=use_vt, runas=user) if env_vars: - if isinstance(env_vars, dict): - for key, val in six.iteritems(env_vars): - if not isinstance(key, six.string_types): - key = str(key) # future lint: disable=blacklisted-function - if not isinstance(val, six.string_types): - val = str(val) # future lint: disable=blacklisted-function - cmd_kwargs.setdefault('env', {})[key] = val - else: - raise CommandExecutionError( - 'env_vars {0} is not a dictionary'.format(env_vars)) + cmd_kwargs.setdefault('env', {}).update(_format_env_vars(env_vars)) try: if cwd: @@ -974,7 +981,8 @@ def uninstall(pkgs=None, def freeze(bin_env=None, user=None, cwd=None, - use_vt=False): + use_vt=False, + env_vars=None): ''' Return a list of installed packages either globally or in the specified virtualenv @@ -1027,6 +1035,8 @@ def freeze(bin_env=None, cmd_kwargs = dict(runas=user, cwd=cwd, use_vt=use_vt, python_shell=False) if bin_env and os.path.isdir(bin_env): cmd_kwargs['env'] = {'VIRTUAL_ENV': bin_env} + if env_vars: + cmd_kwargs.setdefault('env', {}).update(_format_env_vars(env_vars)) result = __salt__['cmd.run_all'](cmd, **cmd_kwargs) if result['retcode'] > 0: @@ -1038,7 +1048,8 @@ def freeze(bin_env=None, def list_(prefix=None, bin_env=None, user=None, - cwd=None): + cwd=None, + env_vars=None): ''' Filter list of installed apps from ``freeze`` and check to see if ``prefix`` exists in the list of packages installed. @@ -1067,7 +1078,7 @@ def list_(prefix=None, if prefix is None or 'pip'.startswith(prefix): packages['pip'] = version(bin_env) - for line in freeze(bin_env=bin_env, user=user, cwd=cwd): + for line in freeze(bin_env=bin_env, user=user, cwd=cwd, env_vars=env_vars): if line.startswith('-f') or line.startswith('#'): # ignore -f line as it contains --find-links directory # ignore comment lines diff --git a/salt/modules/reg.py b/salt/modules/reg.py index 929a63121d..8be6ad04ed 100644 --- a/salt/modules/reg.py +++ b/salt/modules/reg.py @@ -83,6 +83,9 @@ def _to_unicode(vdata): Converts from current users character encoding to unicode. Use this for parameters being pass to reg functions ''' + # None does not convert to Unicode + if vdata is None: + return None return salt.utils.stringutils.to_unicode(vdata, 'utf-8') @@ -526,13 +529,13 @@ def set_value(hive, # https://www.python.org/dev/peps/pep-0237/ # String Types to Unicode - if vtype_value in [1, 2]: + if vtype_value in [win32con.REG_SZ, win32con.REG_EXPAND_SZ]: local_vdata = _to_unicode(vdata) # Don't touch binary... - elif vtype_value == 3: + elif vtype_value == win32con.REG_BINARY: local_vdata = vdata # Make sure REG_MULTI_SZ is a list of strings - elif vtype_value == 7: + elif vtype_value == win32con.REG_MULTI_SZ: local_vdata = [_to_unicode(i) for i in vdata] # Everything else is int else: @@ -686,7 +689,6 @@ def delete_value(hive, key, vname=None, use_32bit_registry=False): salt '*' reg.delete_value HKEY_CURRENT_USER 'SOFTWARE\\Salt' 'version' ''' - local_hive = _to_unicode(hive) local_key = _to_unicode(key) local_vname = _to_unicode(vname) diff --git a/salt/modules/schedule.py b/salt/modules/schedule.py index a77693e66f..0491ee791c 100644 --- a/salt/modules/schedule.py +++ b/salt/modules/schedule.py @@ -58,7 +58,7 @@ SCHEDULE_CONF = [ 'after', 'return_config', 'return_kwargs', - 'run_on_start' + 'run_on_start', 'skip_during_range', 'run_after_skip_range', ] diff --git a/salt/modules/state.py b/salt/modules/state.py index e4972b3715..f357283d26 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -778,19 +778,18 @@ def request(mods=None, 'kwargs': kwargs } }) - cumask = os.umask(0o77) - try: - if salt.utils.platform.is_windows(): - # Make sure cache file isn't read-only - __salt__['cmd.run']('attrib -R "{0}"'.format(notify_path)) - with salt.utils.files.fopen(notify_path, 'w+b') as fp_: - serial.dump(req, fp_) - except (IOError, OSError): - log.error( - 'Unable to write state request file %s. Check permission.', - notify_path - ) - os.umask(cumask) + with salt.utils.files.set_umask(0o077): + try: + if salt.utils.platform.is_windows(): + # Make sure cache file isn't read-only + __salt__['cmd.run']('attrib -R "{0}"'.format(notify_path)) + with salt.utils.files.fopen(notify_path, 'w+b') as fp_: + serial.dump(req, fp_) + except (IOError, OSError): + log.error( + 'Unable to write state request file %s. Check permission.', + notify_path + ) return ret @@ -844,19 +843,18 @@ def clear_request(name=None): req.pop(name) else: return False - cumask = os.umask(0o77) - try: - if salt.utils.platform.is_windows(): - # Make sure cache file isn't read-only - __salt__['cmd.run']('attrib -R "{0}"'.format(notify_path)) - with salt.utils.files.fopen(notify_path, 'w+b') as fp_: - serial.dump(req, fp_) - except (IOError, OSError): - log.error( - 'Unable to write state request file %s. Check permission.', - notify_path - ) - os.umask(cumask) + with salt.utils.files.set_umask(0o077): + try: + if salt.utils.platform.is_windows(): + # Make sure cache file isn't read-only + __salt__['cmd.run']('attrib -R "{0}"'.format(notify_path)) + with salt.utils.files.fopen(notify_path, 'w+b') as fp_: + serial.dump(req, fp_) + except (IOError, OSError): + log.error( + 'Unable to write state request file %s. Check permission.', + notify_path + ) return True @@ -1249,13 +1247,12 @@ def sls(mods, test=None, exclude=None, queue=False, **kwargs): return ['Pillar failed to render with the following messages:'] + errors orchestration_jid = kwargs.get('orchestration_jid') - umask = os.umask(0o77) - if kwargs.get('cache'): - if os.path.isfile(cfn): - with salt.utils.files.fopen(cfn, 'rb') as fp_: - high_ = serial.load(fp_) - return st_.state.call_high(high_, orchestration_jid) - os.umask(umask) + with salt.utils.files.set_umask(0o077): + if kwargs.get('cache'): + if os.path.isfile(cfn): + with salt.utils.files.fopen(cfn, 'rb') as fp_: + high_ = serial.load(fp_) + return st_.state.call_high(high_, orchestration_jid) mods = salt.utils.args.split_input(mods) @@ -1280,36 +1277,36 @@ def sls(mods, test=None, exclude=None, queue=False, **kwargs): if __salt__['config.option']('state_data', '') == 'terse' or kwargs.get('terse'): ret = _filter_running(ret) cache_file = os.path.join(__opts__['cachedir'], 'sls.p') - cumask = os.umask(0o77) - try: - if salt.utils.platform.is_windows(): - # Make sure cache file isn't read-only - __salt__['cmd.run'](['attrib', '-R', cache_file], python_shell=False) - with salt.utils.files.fopen(cache_file, 'w+b') as fp_: - serial.dump(ret, fp_) - except (IOError, OSError): - log.error( - 'Unable to write to SLS cache file %s. Check permission.', - cache_file - ) - _set_retcode(ret, high_) - # Work around Windows multiprocessing bug, set __opts__['test'] back to - # value from before this function was run. - __opts__['test'] = orig_test + with salt.utils.files.set_umask(0o077): + try: + if salt.utils.platform.is_windows(): + # Make sure cache file isn't read-only + __salt__['cmd.run'](['attrib', '-R', cache_file], python_shell=False) + with salt.utils.files.fopen(cache_file, 'w+b') as fp_: + serial.dump(ret, fp_) + except (IOError, OSError): + log.error( + 'Unable to write to SLS cache file %s. Check permission.', + cache_file + ) + _set_retcode(ret, high_) + # Work around Windows multiprocessing bug, set __opts__['test'] back to + # value from before this function was run. + __opts__['test'] = orig_test + + try: + with salt.utils.files.fopen(cfn, 'w+b') as fp_: + try: + serial.dump(high_, fp_) + except TypeError: + # Can't serialize pydsl + pass + except (IOError, OSError): + log.error( + 'Unable to write to highstate cache file %s. Do you have permissions?', + cfn + ) - try: - with salt.utils.files.fopen(cfn, 'w+b') as fp_: - try: - serial.dump(high_, fp_) - except TypeError: - # Can't serialize pydsl - pass - except (IOError, OSError): - log.error( - 'Unable to write to highstate cache file %s. Do you have permissions?', - cfn - ) - os.umask(cumask) _snapper_post(opts, kwargs.get('__pub_jid', 'called localy'), snapper_pre) return ret diff --git a/salt/modules/upstart.py b/salt/modules/upstart.py index 54007734ec..eb6362b2d9 100644 --- a/salt/modules/upstart.py +++ b/salt/modules/upstart.py @@ -98,7 +98,7 @@ def _find_utmp(): result[os.stat(utmp).st_mtime] = utmp except Exception: pass - if result > 0: + if len(result): return result[sorted(result).pop()] else: return False diff --git a/salt/modules/virt.py b/salt/modules/virt.py index 7fb8e16895..385e968c51 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -410,8 +410,7 @@ def _qemu_image_create(vm_name, log.debug('Copying %s to %s', sfn, img_dest) salt.utils.files.copyfile(sfn, img_dest) - mask = os.umask(0) - os.umask(mask) + mask = salt.utils.files.get_umask() if disk_size and qcow2: log.debug('Resize qcow2 image to %sM', disk_size) @@ -433,8 +432,7 @@ def _qemu_image_create(vm_name, else: # Create empty disk try: - mask = os.umask(0) - os.umask(mask) + mask = salt.utils.files.get_umask() if disk_size: log.debug('Create empty image with size %sM', disk_size) diff --git a/salt/modules/win_update.py b/salt/modules/win_update.py deleted file mode 100644 index 778d4803c8..0000000000 --- a/salt/modules/win_update.py +++ /dev/null @@ -1,750 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Module for running windows updates. - -This module is being deprecated and will be removed in Salt Fluorine. Please use -the ``win_wua`` module instead. - -:depends: - win32com - - win32con - - win32api - - pywintypes - -.. versionadded:: 2014.7.0 - -Set windows updates to run by category. Default behavior is to install -all updates that do not require user interaction to complete. -Optionally set ``categories`` to a category of your choice to only -install certain updates. Default is to set to install all available but driver updates. -The following example will install all Security and Critical Updates, -and download but not install standard updates. - -.. code-block:: bash - - salt '*' win_update.install_updates categories="['Critical Updates', 'Security Updates']" - -You can also specify a number of features about the update to have a -fine grain approach to specific types of updates. These are the following -features/states of updates available for configuring: -.. code-block:: text - 'UI' - User interaction required, skipped by default - 'downloaded' - Already downloaded, included by default - 'present' - Present on computer, included by default - 'installed' - Already installed, skipped by default - 'reboot' - Reboot required, included by default - 'hidden' - Skip hidden updates, skipped by default - 'software' - Software updates, included by default - 'driver' - Driver updates, included by default - -The following example installs all updates that don't require a reboot: -.. code-block:: bash - - salt '*' win_update.install_updates skips="[{'reboot':True}]" - - -Once installed Salt will return a similar output: - -.. code-block:: bash - - 2 : Windows Server 2012 Update (KB123456) - 4 : Internet Explorer Security Update (KB098765) - 2 : Malware Definition Update (KB321456) - ... - -The number at the beginning of the line is an OperationResultCode from the Windows Update Agent, -it's enumeration is described here: https://msdn.microsoft.com/en-us/library/windows/desktop/aa387095(v=vs.85).aspx. -The result code is then followed by the update name and its KB identifier. - -''' -# pylint: disable=invalid-name,missing-docstring - -# Import Python libs -from __future__ import absolute_import, unicode_literals, print_function -import logging - -# Import 3rd-party libs -# pylint: disable=import-error -from salt.ext import six -from salt.ext.six.moves import range # pylint: disable=no-name-in-module,redefined-builtin -try: - import win32com.client - import pythoncom - HAS_DEPENDENCIES = True -except ImportError: - HAS_DEPENDENCIES = False -# pylint: enable=import-error - -# Import Salt libs -import salt.utils.platform -import salt.utils.locales -import salt.utils.versions - -log = logging.getLogger(__name__) - - -def __virtual__(): - ''' - Only works on Windows systems - ''' - if salt.utils.platform.is_windows() and HAS_DEPENDENCIES: - salt.utils.versions.warn_until( - 'Fluorine', - 'The \'win_update\' module is being deprecated and will be removed ' - 'in Salt {version}. Please use the \'win_wua\' module instead.' - ) - return True - return (False, "Module win_update: module has failed dependencies or is not on Windows client") - - -def _gather_update_categories(updateCollection): - ''' - this is a convenience method to gather what categories of updates are available in any update - collection it is passed. Typically though, the download_collection. - Some known categories: - Updates - Windows 7 - Critical Updates - Security Updates - Update Rollups - ''' - categories = [] - for i in range(updateCollection.Count): - update = updateCollection.Item(i) - for j in range(update.Categories.Count): - name = update.Categories.Item(j).Name - if name not in categories: - log.debug('found category: %s', name) - categories.append(name) - return categories - - -class PyWinUpdater(object): - def __init__(self, categories=None, skipUI=True, skipDownloaded=False, - skipInstalled=True, skipReboot=False, skipPresent=False, - skipSoftwareUpdates=False, skipDriverUpdates=False, skipHidden=True): - log.debug('CoInitializing the pycom system') - pythoncom.CoInitialize() - - self.skipUI = skipUI - self.skipDownloaded = skipDownloaded - self.skipInstalled = skipInstalled - self.skipReboot = skipReboot - self.skipPresent = skipPresent - self.skipHidden = skipHidden - - self.skipSoftwareUpdates = skipSoftwareUpdates - self.skipDriverUpdates = skipDriverUpdates - - # the list of categories that the user wants to be searched for. - self.categories = categories - - # the list of categories that are present in the updates found. - self.foundCategories = [] - # careful not to get those two confused. - - log.debug('dispatching update_session to keep the session object.') - self.update_session = win32com.client.Dispatch('Microsoft.Update.Session') - - log.debug('update_session got. Now creating a win_searcher to seek out the updates') - self.win_searcher = self.update_session.CreateUpdateSearcher() - - # list of updates that are applicable by current settings. - self.download_collection = win32com.client.Dispatch('Microsoft.Update.UpdateColl') - - # list of updates to be installed. - self.install_collection = win32com.client.Dispatch('Microsoft.Update.UpdateColl') - - # the object responsible for fetching the actual downloads. - self.win_downloader = self.update_session.CreateUpdateDownloader() - self.win_downloader.Updates = self.download_collection - - # the object responsible for the installing of the updates. - self.win_installer = self.update_session.CreateUpdateInstaller() - self.win_installer.Updates = self.install_collection - - # the results of the download process - self.download_results = None - - # the results of the installation process - self.install_results = None - - # search results from CreateUpdateSearcher() - self.search_results = None - - def Search(self, searchString): - try: - log.debug('beginning search of the passed string: %s', searchString) - self.search_results = self.win_searcher.Search(searchString) - log.debug('search completed successfully.') - except Exception as exc: - log.info('search for updates failed. %s', exc) - return exc - - log.debug('parsing results. %s updates were found.', - self.search_results.Updates.Count) - - try: - # step through the list of the updates to ensure that the updates match the - # features desired. - for update in self.search_results.Updates: - # this skipps an update if UI updates are not desired. - if update.InstallationBehavior.CanRequestUserInput: - log.debug(U'Skipped update {0} - requests user input'.format(update.title)) - continue - - # if this update is already downloaded, it doesn't need to be in - # the download_collection. so skipping it unless the user mandates re-download. - if self.skipDownloaded and update.IsDownloaded: - log.debug( - 'Skipped update %s - already downloaded', - update.title - ) - continue - - # check this update's categories against the ones desired. - for category in update.Categories: - # this is a zero guard. these tests have to be in this order - # or it will error out when the user tries to search for - # updates with out specifying categories. - if self.categories is None or category.Name in self.categories: - # adds it to the list to be downloaded. - self.download_collection.Add(update) - log.debug('added update %s', update.title) - # ever update has 2 categories. this prevents the - # from being added twice. - break - log.debug('download_collection made. gathering found categories.') - - # gets the categories of the updates available in this collection of updates - self.foundCategories = _gather_update_categories(self.download_collection) - log.debug('found categories: %s', - six.text_type(self.foundCategories)) - return True - except Exception as exc: - log.info('parsing updates failed. %s', exc) - return exc - - def AutoSearch(self): - ''' - this function generates a search string. simplifying the search function while - still providing as many features as possible. - ''' - search_string = '' - searchParams = [] - - if self.skipInstalled: - searchParams.append('IsInstalled=0') - else: - searchParams.append('IsInstalled=1') - - if self.skipHidden: - searchParams.append('IsHidden=0') - else: - searchParams.append('IsHidden=1') - - if self.skipReboot: - searchParams.append('RebootRequired=0') - else: - searchParams.append('RebootRequired=1') - - if self.skipPresent: - searchParams.append('IsPresent=0') - else: - searchParams.append('IsPresent=1') - - for i in searchParams: - search_string += '{0} and '.format(i) - - if not self.skipSoftwareUpdates and not self.skipDriverUpdates: - search_string += 'Type=\'Software\' or Type=\'Driver\'' - elif not self.skipSoftwareUpdates: - search_string += 'Type=\'Software\'' - elif not self.skipDriverUpdates: - search_string += 'Type=\'Driver\'' - else: - return False - # if there is no type, the is nothing to search. - log.debug('generated search string: %s', search_string) - return self.Search(search_string) - - def Download(self): - # chase the download_collection! do the actual download process. - try: - # if the download_collection is empty. no need to download things. - if self.download_collection.Count != 0: - self.download_results = self.win_downloader.Download() - else: - log.debug('Skipped downloading, all updates were already cached.') - return True - except Exception as exc: - log.debug('failed in the downloading %s.', exc) - return exc - - def Install(self): - # beat those updates into place! - try: - # this does not draw from the download_collection. important thing to know. - # the blugger is created regardless of what the download_collection has done. but it - # will only download those updates which have been downloaded and are ready. - for update in self.search_results.Updates: - if update.IsDownloaded: - self.install_collection.Add(update) - log.debug('Updates prepared. beginning installation') - except Exception as exc: - log.info('Preparing install list failed: %s', exc) - return exc - - # accept eula if not accepted - try: - for update in self.search_results.Updates: - if not update.EulaAccepted: - log.debug('Accepting EULA: %s', update.Title) - update.AcceptEula() - except Exception as exc: - log.info('Accepting Eula failed: %s', exc) - return exc - - # if the blugger is empty. no point it starting the install process. - if self.install_collection.Count != 0: - log.debug('Install list created, about to install') - try: - # the call to install. - self.install_results = self.win_installer.Install() - log.info('Installation of updates complete') - return True - except Exception as exc: - log.info('Installation failed: %s', exc) - return exc - else: - log.info('no new updates.') - return True - - def GetInstallationResults(self): - ''' - this gets results of installation process. - ''' - # if the blugger is empty, the results are nil. - log.debug('blugger has {0} updates in it'.format(self.install_collection.Count)) - if self.install_collection.Count == 0: - return {} - - updates = [] - log.debug('repairing update list') - for i in range(self.install_collection.Count): - # this gets the result from install_results, but the title comes from the update - # collection install_collection. - updates.append('{0}: {1}'.format( - self.install_results.GetUpdateResult(i).ResultCode, - self.install_collection.Item(i).Title)) - - log.debug('Update results enumerated, now making a library to pass back') - results = {} - - # translates the list of update results into a library that salt expects. - for i, update in enumerate(updates): - results['update {0}'.format(i)] = update - - log.debug('Update information complied. returning') - return results - - def GetInstallationResultsPretty(self): - ''' - converts the installation results into a pretty print. - ''' - updates = self.GetInstallationResults() - ret = 'The following are the updates and their return codes.\n' - for i in updates: - ret += '\t{0}\n'.format(updates[i]) - return ret - - def GetDownloadResults(self): - updates = [] - for i in range(self.download_collection.Count): - updates.append('{0}: {1}'.format( - six.text_type(self.download_results.GetUpdateResult(i).ResultCode), - six.text_type(self.download_collection.Item(i).Title))) - results = {} - for i, update in enumerate(updates): - results['update {0}'.format(i)] = update - return results - - def GetSearchResultsVerbose(self): - updates = [] - log.debug('parsing results. %s updates were found.', - self.download_collection.count) - - for update in self.download_collection: - if update.InstallationBehavior.CanRequestUserInput: - log.debug('Skipped update %s', update.title) - continue - # More fields can be added from https://msdn.microsoft.com/en-us/library/windows/desktop/aa386099(v=vs.85).aspx - update_com_fields = ['Categories', 'Deadline', 'Description', - 'Identity', 'IsMandatory', - 'KBArticleIDs', 'MaxDownloadSize', 'MinDownloadSize', - 'MoreInfoUrls', 'MsrcSeverity', 'ReleaseNotes', - 'SecurityBulletinIDs', 'SupportUrl', 'Title'] - simple_enums = ['KBArticleIDs', 'MoreInfoUrls', 'SecurityBulletinIDs'] - # update_dict = {k: getattr(update, k) for k in update_com_fields} - update_dict = {} - for f in update_com_fields: - v = getattr(update, f) - if not any([isinstance(v, bool), isinstance(v, six.string_types)]): - # Fields that require special evaluation. - if f in simple_enums: - v = [x for x in v] - elif f == 'Categories': - v = [{'Name': cat.Name, 'Description': cat.Description} for cat in v] - elif f == 'Deadline': - # Deadline will be useful and should be added. - # However, until it can be tested with a date object - # as returned by the COM, it is unclear how to - # handle this field. - continue - elif f == 'Identity': - v = {'RevisionNumber': v.RevisionNumber, - 'UpdateID': v.UpdateID} - update_dict[f] = v - updates.append(update_dict) - log.debug('added update %s', update.title) - return updates - - def GetSearchResults(self, fields=None): - """Reduce full updates information to the most important information.""" - updates_verbose = self.GetSearchResultsVerbose() - if fields is not None: - updates = [dict((k, v) for k, v in update.items() if k in fields) - for update in updates_verbose] - return updates - # Return list of titles. - return [update['Title'] for update in updates_verbose] - - def SetCategories(self, categories): - self.categories = categories - - def GetCategories(self): - return self.categories - - def GetAvailableCategories(self): - return self.foundCategories - - def SetSkips(self, skips): - if skips: - for i in skips: - value = i[next(six.iterkeys(i))] - skip = next(six.iterkeys(i)) - self.SetSkip(skip, value) - log.debug('was asked to set %s to %s', skip, value) - - def SetSkip(self, skip, state): - if skip == 'UI': - self.skipUI = state - elif skip == 'downloaded': - self.skipDownloaded = state - elif skip == 'installed': - self.skipInstalled = state - elif skip == 'reboot': - self.skipReboot = state - elif skip == 'present': - self.skipPresent = state - elif skip == 'hidden': - self.skipHidden = state - elif skip == 'software': - self.skipSoftwareUpdates = state - elif skip == 'driver': - self.skipDriverUpdates = state - log.debug('new search state: \n\tUI: %s\n\tDownload: %s\n\tInstalled: %s\n\treboot :%s\n\tPresent: %s\n\thidden: %s\n\tsoftware: %s\n\tdriver: %s', - self.skipUI, self.skipDownloaded, self.skipInstalled, self.skipReboot, - self.skipPresent, self.skipHidden, self.skipSoftwareUpdates, self.skipDriverUpdates) - - def __str__(self): - results = 'There are {0} updates, by category there are:\n'.format( - self.download_collection.count) - for category in self.foundCategories: - count = 0 - for update in self.download_collection: - for cat in update.Categories: - if category == cat.Name: - count += 1 - results += '\t{0}: {1}\n'.format(category, count) - return results - - -def _search(quidditch, retries=5): - ''' - a wrapper method for the pywinupdater class. I might move this into the class, but right now, - that is to much for one class I think. - ''' - passed = False - clean = True - comment = '' - while not passed: - log.debug('Searching. tries left: %s', retries) - # let the updater make its own search string. MORE POWER this way. - passed = quidditch.AutoSearch() - log.debug('Done searching: %s', passed) - if isinstance(passed, Exception): - clean = False - comment += 'Failed in the seeking/parsing process:\n\t\t{0}\n'.format(passed) - retries -= 1 - if retries: - comment += '{0} tries to go. retrying\n'.format(str(retries)) - else: - comment += 'out of retries. this update round failed.\n' - return (comment, True, retries) - passed = False - if clean: - # bragging rights. - comment += 'Search was done without error.\n' - - return (comment, True, retries) - - -def _download(quidditch, retries=5): - ''' - another wrapper method. - ''' - passed = False - clean = True - comment = '' - while not passed: - log.debug('Downloading. tries left: %s', retries) - passed = quidditch.Download() - log.debug('Done downloading: %s', passed) - if isinstance(passed, Exception): - clean = False - comment += 'Failed while trying to download updates:\n\t\t{0}\n'.format(str(passed)) - retries -= 1 - if retries: - comment += '{0} tries to go. retrying\n'.format(str(retries)) - passed = False - else: - comment += 'out of retries. this update round failed.\n' - return (comment, False, retries) - if clean: - comment += 'Download was done without error.\n' - return (comment, True, retries) - - -def _install(quidditch, retries=5): - ''' - and the last wrapper method. keeping things simple. - ''' - passed = False - clean = True - comment = '' - while not passed: - log.debug('download_collection is this long: %s', - quidditch.install_collection.Count) - log.debug('Installing. tries left: %s', retries) - passed = quidditch.Install() - log.info('Done installing: %s', passed) - if isinstance(passed, Exception): - clean = False - comment += 'Failed while trying to install the updates.\n\t\t{0}\n'.format(str(passed)) - retries -= 1 - if retries: - comment += '{0} tries to go. retrying\n'.format(str(retries)) - passed = False - else: - comment += 'out of retries. this update round failed.\n' - return (comment, False, retries) - if clean: - comment += 'Install was done without error.\n' - return (comment, True, retries) - - -# this is where the actual functions available to salt begin. - -def list_updates(verbose=False, fields=None, skips=None, retries=5, categories=None): - ''' - Returns a summary of available updates, grouped into their non-mutually - exclusive categories. - - verbose - Return full set of results, including several fields from the COM. - - fields - Return a list of specific fields for each update. The optional - values here are those at the root level of the verbose list. This - is superseded by the verbose option. - - retries - Number of retries to make before giving up. This is total, not per - step. - - categories - Specify the categories to list. Must be passed as a list. - - .. code-block:: bash - - salt '*' win_update.list_updates categories="['Updates']" - - Categories include, but are not limited to, the following: - - * Updates - * Windows 7 - * Critical Updates - * Security Updates - * Update Rollups - - CLI Examples: - - .. code-block:: bash - - # Normal Usage - salt '*' win_update.list_updates - - # Specific Fields - salt '*' win_update.list_updates fields="['Title', 'Description']" - - # List all critical updates list in verbose detail - salt '*' win_update.list_updates categories="['Critical Updates']" verbose=True - - ''' - - log.debug('categories to search for are: %s', categories) - updates = PyWinUpdater() - if categories: - updates.SetCategories(categories) - updates.SetSkips(skips) - - # this is where we be seeking the things! yar! - comment, passed, retries = _search(updates, retries) - if not passed: - return (comment, str(passed)) - log.debug('verbose: %s', verbose) - if verbose: - return updates.GetSearchResultsVerbose() - return updates.GetSearchResults(fields=fields) - - -def download_updates(skips=None, retries=5, categories=None): - ''' - Downloads all available updates, skipping those that require user - interaction. - - Various aspects of the updates can be included or excluded. this feature is - still in development. - - retries - Number of retries to make before giving up. This is total, not per - step. - - categories - Specify the categories to update. Must be passed as a list. - - .. code-block:: bash - - salt '*' win_update.download_updates categories="['Updates']" - - Categories include the following: - - * Updates - * Windows 7 - * Critical Updates - * Security Updates - * Update Rollups - - CLI Examples: - - .. code-block:: bash - - # Normal Usage - salt '*' win_update.download_updates - - # Download critical updates only - salt '*' win_update.download_updates categories="['Critical Updates']" - - ''' - - log.debug('categories to search for are: %s', categories) - quidditch = PyWinUpdater(skipDownloaded=True) - quidditch.SetCategories(categories) - quidditch.SetSkips(skips) - - # this is where we be seeking the things! yar! - comment, passed, retries = _search(quidditch, retries) - if not passed: - return (comment, str(passed)) - - # this is where we get all the things! i.e. download updates. - comment, passed, retries = _download(quidditch, retries) - if not passed: - return (comment, str(passed)) - - try: - comment = quidditch.GetDownloadResults() - except Exception as exc: - comment = 'could not get results, but updates were installed. {0}'.format(exc) - return 'Windows is up to date. \n{0}'.format(comment) - - -def install_updates(skips=None, retries=5, categories=None): - ''' - Downloads and installs all available updates, skipping those that require - user interaction. - - Add ``cached`` to only install those updates which have already been downloaded. - - you can set the maximum number of retries to ``n`` in the search process by - adding: ``retries=n`` - - various aspects of the updates can be included or excluded. This function is - still under development. - - retries - Number of retries to make before giving up. This is total, not per - step. - - categories - Specify the categories to install. Must be passed as a list. - - .. code-block:: bash - - salt '*' win_update.install_updates categories="['Updates']" - - Categories include the following: - - * Updates - * Windows 7 - * Critical Updates - * Security Updates - * Update Rollups - - CLI Examples: - - .. code-block:: bash - - # Normal Usage - salt '*' win_update.install_updates - - # Install all critical updates - salt '*' win_update.install_updates categories="['Critical Updates']" - - ''' - - log.debug('categories to search for are: %s', categories) - quidditch = PyWinUpdater() - quidditch.SetCategories(categories) - quidditch.SetSkips(skips) - - # this is where we be seeking the things! yar! - comment, passed, retries = _search(quidditch, retries) - if not passed: - return (comment, str(passed)) - - # this is where we get all the things! i.e. download updates. - comment, passed, retries = _download(quidditch, retries) - if not passed: - return (comment, str(passed)) - - # this is where we put things in their place! - comment, passed, retries = _install(quidditch, retries) - if not passed: - return (comment, str(passed)) - - try: - comment = quidditch.GetInstallationResultsPretty() - except Exception as exc: - comment = 'Could not get results, but updates were installed. {0}'.format(exc) - return 'Windows is up to date. \n{0}'.format(comment) diff --git a/salt/modules/win_wua.py b/salt/modules/win_wua.py index 222a35ec21..b37d7676ca 100644 --- a/salt/modules/win_wua.py +++ b/salt/modules/win_wua.py @@ -56,7 +56,6 @@ import logging # Import Salt libs import salt.utils.platform -import salt.utils.versions import salt.utils.win_update from salt.exceptions import CommandExecutionError @@ -228,87 +227,6 @@ def available(software=True, return updates.summary() if summary else updates.list() -def list_update(name, download=False, install=False): - ''' - .. deprecated:: 2017.7.0 - Use :func:`get` instead - - Returns details for all updates that match the search criteria - - Args: - - name (str): - The name of the update you're searching for. This can be the GUID, a - KB number, or any part of the name of the update. GUIDs and KBs are - preferred. Run ``list_updates`` to get the GUID for the update - you're looking for. - - download (bool): - Download the update returned by this function. Run this function - first to see if the update exists, then set ``download=True`` to - download the update. - - install (bool): - Install the update returned by this function. Run this function - first to see if the update exists, then set ``install=True`` to - install the update. - - Returns: - - dict: Returns a dict containing a list of updates that match the name if - download and install are both set to False. Should usually be a single - update, but can return multiple if a partial name is given. - - If download or install is set to true it will return the results of the - operation. - - .. code-block:: cfg - - List of Updates: - {'': {'Title': , - 'KB': <KB>, - 'GUID': <the globally unique identifier for the update> - 'Description': <description>, - 'Downloaded': <has the update been downloaded>, - 'Installed': <has the update been installed>, - 'Mandatory': <is the update mandatory>, - 'UserInput': <is user input required>, - 'EULAAccepted': <has the EULA been accepted>, - 'Severity': <update severity>, - 'NeedsReboot': <is the update installed and awaiting reboot>, - 'RebootBehavior': <will the update require a reboot>, - 'Categories': [ '<category 1>', - '<category 2>', - ...] - } - } - - CLI Examples: - - .. code-block:: bash - - # Recommended Usage using GUID without braces - # Use this to find the status of a specific update - salt '*' win_wua.list_update 12345678-abcd-1234-abcd-1234567890ab - - # Use the following if you don't know the GUID: - - # Using a KB number (could possibly return multiple results) - # Not all updates have an associated KB - salt '*' win_wua.list_update KB3030298 - - # Using part or all of the name of the update - # Could possibly return multiple results - # Not all updates have an associated KB - salt '*' win_wua.list_update 'Microsoft Camera Codec Pack' - ''' - salt.utils.versions.warn_until( - 'Fluorine', - 'This function is replaced by \'get\' as of Salt 2017.7.0. This ' - 'warning will be removed in Salt Fluorine.') - return get(name, download, install) - - def get(name, download=False, install=False): ''' .. versionadded:: 2017.7.0 @@ -401,142 +319,6 @@ def get(name, download=False, install=False): return ret if ret else updates.list() -def list_updates(software=True, - drivers=False, - summary=False, - skip_installed=True, - categories=None, - severities=None, - download=False, - install=False): - ''' - .. deprecated:: 2017.7.0 - Use :func:`list` instead - - Returns a detailed list of available updates or a summary. If download or - install is True the same list will be downloaded and/or installed. - - Args: - - software (bool): - Include software updates in the results (default is True) - - drivers (bool): - Include driver updates in the results (default is False) - - summary (bool): - - True: Return a summary of updates available for each category. - - False (default): Return a detailed list of available updates. - - skip_installed (bool): - Skip installed updates in the results (default is False) - - download (bool): - (Overrides reporting functionality) Download the list of updates - returned by this function. Run this function first with - ``download=False`` to see what will be downloaded, then set - ``download=True`` to download the updates. - - install (bool): - (Overrides reporting functionality) Install the list of updates - returned by this function. Run this function first with - ``install=False`` to see what will be installed, then set - ``install=True`` to install the updates. - - categories (list): - Specify the categories to list. Must be passed as a list. All - categories returned by default. - - Categories include the following: - - * Critical Updates - * Definition Updates - * Drivers (make sure you set drivers=True) - * Feature Packs - * Security Updates - * Update Rollups - * Updates - * Update Rollups - * Windows 7 - * Windows 8.1 - * Windows 8.1 drivers - * Windows 8.1 and later drivers - * Windows Defender - - severities (list): - Specify the severities to include. Must be passed as a list. All - severities returned by default. - - Severities include the following: - - * Critical - * Important - - Returns: - - dict: Returns a dict containing either a summary or a list of updates: - - .. code-block:: cfg - - List of Updates: - {'<GUID>': {'Title': <title>, - 'KB': <KB>, - 'GUID': <the globally unique identifier for the update> - 'Description': <description>, - 'Downloaded': <has the update been downloaded>, - 'Installed': <has the update been installed>, - 'Mandatory': <is the update mandatory>, - 'UserInput': <is user input required>, - 'EULAAccepted': <has the EULA been accepted>, - 'Severity': <update severity>, - 'NeedsReboot': <is the update installed and awaiting reboot>, - 'RebootBehavior': <will the update require a reboot>, - 'Categories': [ '<category 1>', - '<category 2>', - ...] - } - } - - Summary of Updates: - {'Total': <total number of updates returned>, - 'Available': <updates that are not downloaded or installed>, - 'Downloaded': <updates that are downloaded but not installed>, - 'Installed': <updates installed (usually 0 unless installed=True)>, - 'Categories': { <category 1>: <total for that category>, - <category 2>: <total for category 2>, - ... } - } - - CLI Examples: - - .. code-block:: bash - - # Normal Usage (list all software updates) - salt '*' win_wua.list_updates - - # List all updates with categories of Critical Updates and Drivers - salt '*' win_wua.list_updates categories=['Critical Updates','Drivers'] - - # List all Critical Security Updates - salt '*' win_wua.list_updates categories=['Security Updates'] severities=['Critical'] - - # List all updates with a severity of Critical - salt '*' win_wua.list_updates severities=['Critical'] - - # A summary of all available updates - salt '*' win_wua.list_updates summary=True - - # A summary of all Feature Packs and Windows 8.1 Updates - salt '*' win_wua.list_updates categories=['Feature Packs','Windows 8.1'] summary=True - ''' - salt.utils.versions.warn_until( - 'Fluorine', - 'This function is replaced by \'list\' as of Salt 2017.7.0. This ' - 'warning will be removed in Salt Fluorine.') - return list(software, drivers, summary, skip_installed, categories, - severities, download, install) - - def list(software=True, drivers=False, summary=False, @@ -688,74 +470,6 @@ def list(software=True, return ret -def download_update(name): - ''' - .. deprecated:: 2017.7.0 - Use :func:`download` instead - - Downloads a single update. - - Args: - - name (str): - The name of the update to download. This can be a GUID, a KB number, - or any part of the name. To ensure a single item is matched the GUID - is preferred. - - .. note:: - If more than one result is returned an error will be raised. - - Returns: - - dict: A dictionary containing the results of the download - - CLI Examples: - - .. code-block:: bash - - salt '*' win_wua.download_update 12345678-abcd-1234-abcd-1234567890ab - - salt '*' win_wua.download_update KB12312321 - ''' - salt.utils.versions.warn_until( - 'Fluorine', - 'This function is replaced by \'download\' as of Salt 2017.7.0. This ' - 'warning will be removed in Salt Fluorine.') - return download(name) - - -def download_updates(names): - ''' - .. deprecated:: 2017.7.0 - Use :func:`download` instead - - Downloads updates that match the list of passed identifiers. It's easier to - use this function by using list_updates and setting install=True. - - Args: - - names (list): - A list of updates to download. This can be any combination of GUIDs, - KB numbers, or names. GUIDs or KBs are preferred. - - Returns: - - dict: A dictionary containing the details about the downloaded updates - - CLI Examples: - - .. code-block:: bash - - # Normal Usage - salt '*' win_wua.download_updates guid=['12345678-abcd-1234-abcd-1234567890ab', 'KB2131233'] - ''' - salt.utils.versions.warn_until( - 'Fluorine', - 'This function is replaced by \'download\' as of Salt 2017.7.0. This ' - 'warning will be removed in Salt Fluorine.') - return download(names) - - def download(names): ''' .. versionadded:: 2017.7.0 @@ -808,73 +522,6 @@ def download(names): return wua.download(updates) -def install_update(name): - ''' - .. deprecated:: 2017.7.0 - Use :func:`install` instead - - Installs a single update - - Args: - - name (str): The name of the update to install. This can be a GUID, a KB - number, or any part of the name. To ensure a single item is matched the - GUID is preferred. - - .. note:: - If no results or more than one result is returned an error will be - raised. - - Returns: - - dict: A dictionary containing the results of the install - - CLI Examples: - - .. code-block:: bash - - salt '*' win_wua.install_update 12345678-abcd-1234-abcd-1234567890ab - - salt '*' win_wua.install_update KB12312231 - ''' - salt.utils.versions.warn_until( - 'Fluorine', - 'This function is replaced by \'install\' as of Salt 2017.7.0. This ' - 'warning will be removed in Salt Fluorine.') - return install(name) - - -def install_updates(names): - ''' - .. deprecated:: 2017.7.0 - Use :func:`install` instead - - Installs updates that match the list of identifiers. It may be easier to use - the list_updates function and set install=True. - - Args: - - names (list): A list of updates to install. This can be any combination - of GUIDs, KB numbers, or names. GUIDs or KBs are preferred. - - Returns: - - dict: A dictionary containing the details about the installed updates - - CLI Examples: - - .. code-block:: bash - - # Normal Usage - salt '*' win_wua.install_updates guid=['12345678-abcd-1234-abcd-1234567890ab', 'KB12323211'] - ''' - salt.utils.versions.warn_until( - 'Fluorine', - 'This function is replaced by \'install\' as of Salt 2017.7.0. This ' - 'warning will be removed in Salt Fluorine.') - return install(names) - - def install(names): ''' .. versionadded:: 2017.7.0 diff --git a/salt/modules/x509.py b/salt/modules/x509.py index 108379635e..f741cf085e 100644 --- a/salt/modules/x509.py +++ b/salt/modules/x509.py @@ -756,28 +756,27 @@ def write_pem(text, path, overwrite=True, pem_type=None): "-----BEGIN CERTIFICATE-----MIIGMzCCBBugA..." \\ path=/etc/pki/mycert.crt ''' - old_umask = os.umask(0o77) - text = get_pem_entry(text, pem_type=pem_type) - _dhparams = '' - _private_key = '' - if pem_type and pem_type == 'CERTIFICATE' and os.path.isfile(path) and \ - not overwrite: - _filecontents = _text_or_file(path) - try: - _dhparams = get_pem_entry(_filecontents, 'DH PARAMETERS') - except salt.exceptions.SaltInvocationError: - pass - try: - _private_key = get_pem_entry(_filecontents, '(?:RSA )?PRIVATE KEY') - except salt.exceptions.SaltInvocationError: - pass - with salt.utils.files.fopen(path, 'w') as _fp: - if pem_type and pem_type == 'CERTIFICATE' and _private_key: - _fp.write(salt.utils.stringutils.to_str(_private_key)) - _fp.write(text) - if pem_type and pem_type == 'CERTIFICATE' and _dhparams: - _fp.write(salt.utils.stringutils.to_str(_dhparams)) - os.umask(old_umask) + with salt.utils.files.set_umask(0o077): + text = get_pem_entry(text, pem_type=pem_type) + _dhparams = '' + _private_key = '' + if pem_type and pem_type == 'CERTIFICATE' and os.path.isfile(path) and \ + not overwrite: + _filecontents = _text_or_file(path) + try: + _dhparams = get_pem_entry(_filecontents, 'DH PARAMETERS') + except salt.exceptions.SaltInvocationError: + pass + try: + _private_key = get_pem_entry(_filecontents, '(?:RSA )?PRIVATE KEY') + except salt.exceptions.SaltInvocationError: + pass + with salt.utils.files.fopen(path, 'w') as _fp: + if pem_type and pem_type == 'CERTIFICATE' and _private_key: + _fp.write(salt.utils.stringutils.to_str(_private_key)) + _fp.write(text) + if pem_type and pem_type == 'CERTIFICATE' and _dhparams: + _fp.write(salt.utils.stringutils.to_str(_dhparams)) return 'PEM written to {0}'.format(path) diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py index 7f927e9473..a2f8bc8115 100644 --- a/salt/netapi/rest_tornado/saltnado.py +++ b/salt/netapi/rest_tornado/saltnado.py @@ -307,9 +307,9 @@ class EventListener(object): ''' if request not in self.request_map: return - for tag, future in self.request_map[request]: + for tag, matcher, future in self.request_map[request]: # timeout the future - self._timeout_future(tag, future) + self._timeout_future(tag, matcher, future) # remove the timeout if future in self.timeout_map: tornado.ioloop.IOLoop.current().remove_timeout(self.timeout_map[future]) @@ -317,9 +317,22 @@ class EventListener(object): del self.request_map[request] + @staticmethod + def prefix_matcher(mtag, tag): + if mtag is None or tag is None: + raise TypeError('mtag or tag can not be None') + return mtag.startswith(tag) + + @staticmethod + def exact_matcher(mtag, tag): + if mtag is None or tag is None: + raise TypeError('mtag or tag can not be None') + return mtag == tag + def get_event(self, request, tag='', + matcher=prefix_matcher.__func__, callback=None, timeout=None ): @@ -339,43 +352,52 @@ class EventListener(object): tornado.ioloop.IOLoop.current().add_callback(callback, future) future.add_done_callback(handle_future) # add this tag and future to the callbacks - self.tag_map[tag].append(future) - self.request_map[request].append((tag, future)) + self.tag_map[(tag, matcher)].append(future) + self.request_map[request].append((tag, matcher, future)) if timeout: - timeout_future = tornado.ioloop.IOLoop.current().call_later(timeout, self._timeout_future, tag, future) + timeout_future = tornado.ioloop.IOLoop.current().call_later(timeout, self._timeout_future, tag, matcher, future) self.timeout_map[future] = timeout_future return future - def _timeout_future(self, tag, future): + def _timeout_future(self, tag, matcher, future): ''' Timeout a specific future ''' - if tag not in self.tag_map: + if (tag, matcher) not in self.tag_map: return if not future.done(): future.set_exception(TimeoutException()) - self.tag_map[tag].remove(future) - if len(self.tag_map[tag]) == 0: - del self.tag_map[tag] + self.tag_map[(tag, matcher)].remove(future) + if len(self.tag_map[(tag, matcher)]) == 0: + del self.tag_map[(tag, matcher)] def _handle_event_socket_recv(self, raw): ''' Callback for events on the event sub socket ''' mtag, data = self.event.unpack(raw, self.event.serial) + # see if we have any futures that need this info: - for tag_prefix, futures in six.iteritems(self.tag_map): - if mtag.startswith(tag_prefix): - for future in futures: - if future.done(): - continue - future.set_result({'data': data, 'tag': mtag}) - self.tag_map[tag_prefix].remove(future) - if future in self.timeout_map: - tornado.ioloop.IOLoop.current().remove_timeout(self.timeout_map[future]) - del self.timeout_map[future] + for (tag, matcher), futures in six.iteritems(self.tag_map): + try: + is_matched = matcher(mtag, tag) + except Exception as e: + log.error('Failed to run a matcher.', exc_info=True) + is_matched = False + + if not is_matched: + continue + + for future in futures: + if future.done(): + continue + future.set_result({'data': data, 'tag': mtag}) + self.tag_map[(tag, matcher)].remove(future) + if future in self.timeout_map: + tornado.ioloop.IOLoop.current().remove_timeout(self.timeout_map[future]) + del self.timeout_map[future] class BaseSaltAPIHandler(tornado.web.RequestHandler): # pylint: disable=W0223 @@ -924,64 +946,83 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 if self.application.opts['order_masters']: syndic_min_wait = tornado.gen.sleep(self.application.opts['syndic_wait']) - job_not_running = self.job_not_running(pub_data['jid'], - chunk['tgt'], - f_call['kwargs']['tgt_type'], - minions_remaining=minions_remaining - ) + # To ensure job_not_running and all_return are terminated by each other, communicate using a future + is_finished = Future() + job_not_running_future = self.job_not_running(pub_data['jid'], + chunk['tgt'], + f_call['kwargs']['tgt_type'], + is_finished, + minions_remaining=list(minions_remaining), + ) # if we have a min_wait, do that if syndic_min_wait is not None: yield syndic_min_wait - # we are completed when either all minions return or the job isn't running anywhere - chunk_ret = yield self.all_returns(pub_data['jid'], - finish_futures=[job_not_running], - minions_remaining=minions_remaining, - ) - raise tornado.gen.Return(chunk_ret) + all_return_future = self.all_returns(pub_data['jid'], + is_finished, + minions_remaining=list(minions_remaining), + ) + yield job_not_running_future + raise tornado.gen.Return((yield all_return_future)) @tornado.gen.coroutine def all_returns(self, jid, - finish_futures=None, + is_finished, minions_remaining=None, ): ''' Return a future which will complete once all returns are completed - (according to minions_remaining), or one of the passed in "finish_futures" completes + (according to minions_remaining), or one of the passed in "is_finished" completes ''' - if finish_futures is None: - finish_futures = [] if minions_remaining is None: minions_remaining = [] - ret_tag = tagify([jid, 'ret'], 'job') chunk_ret = {} + + minion_events = {} + for minion in minions_remaining: + tag = tagify([jid, 'ret', minion], 'job') + minion_event = self.application.event_listener.get_event(self, + tag=tag, + matcher=EventListener.exact_matcher, + timeout=self.application.opts['timeout']) + minion_events[minion_event] = minion + while True: - ret_event = self.application.event_listener.get_event(self, - tag=ret_tag, - ) - f = yield Any([ret_event] + finish_futures) - if f in finish_futures: - raise tornado.gen.Return(chunk_ret) - event = f.result() - chunk_ret[event['data']['id']] = event['data']['return'] - # its possible to get a return that wasn't in the minion_remaining list + f = yield Any(minion_events.keys() + [is_finished]) try: - minions_remaining.remove(event['data']['id']) + if f is is_finished: + for event in minion_events: + if not event.done(): + event.set_result(None) + raise tornado.gen.Return(chunk_ret) + f_result = f.result() + chunk_ret[f_result['data']['id']] = f_result['data']['return'] + except TimeoutException: + pass + + # clear finished event future + try: + minions_remaining.remove(minion_events[f]) + del minion_events[f] except ValueError: pass + if len(minions_remaining) == 0: + if not is_finished.done(): + is_finished.set_result(True) raise tornado.gen.Return(chunk_ret) @tornado.gen.coroutine def job_not_running(self, - jid, - tgt, - tgt_type, - minions_remaining=None, - ): + jid, + tgt, + tgt_type, + is_finished, + minions_remaining=None, + ): ''' Return a future which will complete once jid (passed in) is no longer running on tgt @@ -998,12 +1039,21 @@ class SaltAPIHandler(BaseSaltAPIHandler): # pylint: disable=W0223 minion_running = False while True: try: - event = yield self.application.event_listener.get_event(self, - tag=ping_tag, - timeout=self.application.opts['gather_job_timeout'], - ) + event = self.application.event_listener.get_event(self, + tag=ping_tag, + timeout=self.application.opts['gather_job_timeout'], + ) + f = yield Any([event, is_finished]) + # When finished entire routine, cleanup other futures and return result + if f is is_finished: + if not event.done(): + event.set_result(None) + raise tornado.gen.Return(True) + event = f.result() except TimeoutException: if not minion_running: + if not is_finished.done(): + is_finished.set_result(True) raise tornado.gen.Return(True) else: ping_pub_data = yield self.saltclients['local'](tgt, diff --git a/salt/runners/nacl.py b/salt/runners/nacl.py index 53af19ec20..5727356258 100644 --- a/salt/runners/nacl.py +++ b/salt/runners/nacl.py @@ -8,9 +8,6 @@ This is often useful if you wish to store your pillars in source control or share your pillar data with others that you trust. I don't advise making your pillars public regardless if they are encrypted or not. -When generating keys and encrypting passwords use --local when using salt-call for extra -security. Also consider using just the salt runner nacl when encrypting pillar passwords. - :configuration: The following configuration defaults can be define (pillar or config files) Avoid storing private keys in pillars! Ensure master does not have `pillar_opts=True`: @@ -30,7 +27,7 @@ security. Also consider using just the salt runner nacl when encrypting pillar p .. code-block:: bash - salt-call nacl.enc sk_file=/etc/salt/pki/master/nacl pk_file=/etc/salt/pki/master/nacl.pub + salt-run nacl.enc sk_file=/etc/salt/pki/master/nacl pk_file=/etc/salt/pki/master/nacl.pub The nacl lib uses 32byte keys, these keys are base64 encoded to make your life more simple. @@ -38,9 +35,9 @@ To generate your `sk_file` and `pk_file` use: .. code-block:: bash - salt-call --local nacl.keygen sk_file=/etc/salt/pki/master/nacl + salt-run nacl.keygen sk_file=/etc/salt/pki/master/nacl # or if you want to work without files. - salt-call --local nacl.keygen + salt-run nacl.keygen local: ---------- pk: @@ -59,14 +56,14 @@ Sealedbox only has one key that is for both encryption and decryption. .. code-block:: bash - salt-call --local nacl.enc asecretpass pk=/kfGX7PbWeu099702PBbKWLpG/9p06IQRswkdWHCDk0= + salt-run nacl.enc asecretpass pk=/kfGX7PbWeu099702PBbKWLpG/9p06IQRswkdWHCDk0= tqXzeIJnTAM9Xf0mdLcpEdklMbfBGPj2oTKmlgrm3S1DTVVHNnh9h8mU1GKllGq/+cYsk6m5WhGdk58= To decrypt the data: .. code-block:: bash - salt-call --local nacl.dec data='tqXzeIJnTAM9Xf0mdLcpEdklMbfBGPj2oTKmlgrm3S1DTVVHNnh9h8mU1GKllGq/+cYsk6m5WhGdk58=' \ + salt-run nacl.dec data='tqXzeIJnTAM9Xf0mdLcpEdklMbfBGPj2oTKmlgrm3S1DTVVHNnh9h8mU1GKllGq/+cYsk6m5WhGdk58=' \ sk='SVWut5SqNpuPeNzb1b9y6b2eXg2PLIog43GBzp48Sow=' When the keys are defined in the master config you can use them from the nacl runner @@ -94,7 +91,7 @@ The developer can then use a less-secure system to encrypt data. .. code-block:: bash - salt-call --local nacl.enc apassword + salt-run nacl.enc apassword Pillar files can include protected data that the salt master decrypts: @@ -111,42 +108,7 @@ Larger files like certificates can be encrypted with: .. code-block:: bash - salt-call nacl.enc_file /tmp/cert.crt out=/tmp/cert.nacl - # or more advanced - cert=$(cat /tmp/cert.crt) - salt-call --out=newline_values_only nacl.enc_pub data="$cert" > /tmp/cert.nacl - -In pillars rended with jinja be sure to include `|json` so line breaks are encoded: - -.. code-block:: jinja - - cert: "{{salt.nacl.dec('S2uogToXkgENz9...085KYt')|json}}" - -In states rendered with jinja it is also good pratice to include `|json`: - -.. code-block:: jinja - - {{sls}} private key: - file.managed: - - name: /etc/ssl/private/cert.key - - mode: 700 - - contents: "{{pillar['pillarexample']['cert_key']|json}}" - - -Optional small program to encrypt data without needing salt modules. - -.. code-block:: python - - #!/bin/python3 - import sys, base64, libnacl.sealed - pk = base64.b64decode('YOURPUBKEY') - b = libnacl.sealed.SealedBox(pk) - data = sys.stdin.buffer.read() - print(base64.b64encode(b.encrypt(data)).decode()) - -.. code-block:: bash - - echo 'apassword' | nacl_enc.py + salt-run nacl.enc_file /tmp/cert.crt out=/tmp/cert.nacl ''' @@ -158,6 +120,7 @@ import os # Import Salt libs import salt.utils.files import salt.utils.platform +import salt.utils.stringutils import salt.utils.win_functions import salt.utils.win_dacl import salt.syspaths @@ -186,9 +149,9 @@ def _get_config(**kwargs): config = { 'box_type': 'sealedbox', 'sk': None, - 'sk_file': '/etc/salt/pki/master/nacl', + 'sk_file': os.path.join(__opts__['pki_dir'], 'nacl'), 'pk': None, - 'pk_file': '/etc/salt/pki/master/nacl.pub', + 'pk_file': os.path.join(__opts__['pki_dir'], 'nacl.pub'), } config_key = '{0}.config'.format(__virtualname__) try: @@ -233,7 +196,7 @@ def _get_pk(**kwargs): return base64.b64decode(pubkey) -def keygen(sk_file=None, pk_file=None): +def keygen(sk_file=None, pk_file=None, **kwargs): ''' Use libnacl to generate a keypair. @@ -248,11 +211,20 @@ def keygen(sk_file=None, pk_file=None): .. code-block:: bash - salt-call nacl.keygen - salt-call nacl.keygen sk_file=/etc/salt/pki/master/nacl - salt-call nacl.keygen sk_file=/etc/salt/pki/master/nacl pk_file=/etc/salt/pki/master/nacl.pub - salt-call --local nacl.keygen + salt-run nacl.keygen + salt-run nacl.keygen sk_file=/etc/salt/pki/master/nacl + salt-run nacl.keygen sk_file=/etc/salt/pki/master/nacl pk_file=/etc/salt/pki/master/nacl.pub + salt-run nacl.keygen ''' + + if 'keyfile' in kwargs: + salt.utils.versions.warn_until( + 'Fluorine', + 'The \'keyfile\' argument has been deprecated and will be removed in Salt ' + '{version}. Please use \'sk_file\' argument instead.' + ) + sk_file = kwargs['keyfile'] + if sk_file is None: kp = libnacl.public.SecretKey() return {'sk': base64.b64encode(kp.sk), 'pk': base64.b64encode(kp.pk)} @@ -313,6 +285,26 @@ def enc(data, **kwargs): box_type: secretbox, sealedbox(default) ''' + + if 'keyfile' in kwargs: + salt.utils.versions.warn_until( + 'Fluorine', + 'The \'keyfile\' argument has been deprecated and will be removed in Salt ' + '{version}. Please use \'sk_file\' argument instead.' + ) + kwargs['sk_file'] = kwargs['keyfile'] + + if 'key' in kwargs: + salt.utils.versions.warn_until( + 'Fluorine', + 'The \'key\' argument has been deprecated and will be removed in Salt ' + '{version}. Please use \'sk\' argument instead.' + ) + kwargs['sk'] = kwargs['key'] + + # ensure data is bytes + data = salt.utils.stringutils.to_bytes(data) + box_type = _get_config(**kwargs)['box_type'] if box_type == 'sealedbox': return sealedbox_encrypt(data, **kwargs) @@ -334,7 +326,6 @@ def enc_file(name, out=None, **kwargs): .. code-block:: bash salt-run nacl.enc_file name=/tmp/id_rsa - salt-call nacl.enc_file name=salt://crt/mycert out=/tmp/cert salt-run nacl.enc_file name=/tmp/id_rsa box_type=secretbox \ sk_file=/etc/salt/pki/master/nacl.pub ''' @@ -360,6 +351,31 @@ def dec(data, **kwargs): box_type: secretbox, sealedbox(default) ''' + if 'keyfile' in kwargs: + salt.utils.versions.warn_until( + 'Fluorine', + 'The \'keyfile\' argument has been deprecated and will be removed in Salt ' + '{version}. Please use \'sk_file\' argument instead.' + ) + kwargs['sk_file'] = kwargs['keyfile'] + + # set boxtype to `secretbox` to maintain backward compatibility + kwargs['box_type'] = 'secretbox' + + if 'key' in kwargs: + salt.utils.versions.warn_until( + 'Fluorine', + 'The \'key\' argument has been deprecated and will be removed in Salt ' + '{version}. Please use \'sk\' argument instead.' + ) + kwargs['sk'] = kwargs['key'] + + # set boxtype to `secretbox` to maintain backward compatibility + kwargs['box_type'] = 'secretbox' + + # ensure data is bytes + data = salt.utils.stringutils.to_bytes(data) + box_type = _get_config(**kwargs)['box_type'] if box_type == 'sealedbox': return sealedbox_decrypt(data, **kwargs) @@ -381,7 +397,6 @@ def dec_file(name, out=None, **kwargs): .. code-block:: bash salt-run nacl.dec_file name=/tmp/id_rsa.nacl - salt-call nacl.dec_file name=salt://crt/mycert.nacl out=/tmp/id_rsa salt-run nacl.dec_file name=/tmp/id_rsa.nacl box_type=secretbox \ sk_file=/etc/salt/pki/master/nacl.pub ''' @@ -411,9 +426,10 @@ def sealedbox_encrypt(data, **kwargs): .. code-block:: bash salt-run nacl.sealedbox_encrypt datatoenc - salt-call --local nacl.sealedbox_encrypt datatoenc pk_file=/etc/salt/pki/master/nacl.pub - salt-call --local nacl.sealedbox_encrypt datatoenc pk='vrwQF7cNiNAVQVAiS3bvcbJUnF0cN6fU9YTZD9mBfzQ=' ''' + # ensure data is bytes + data = salt.utils.stringutils.to_bytes(data) + pk = _get_pk(**kwargs) b = libnacl.sealed.SealedBox(pk) return base64.b64encode(b.encrypt(data)) @@ -427,12 +443,16 @@ def sealedbox_decrypt(data, **kwargs): .. code-block:: bash - salt-call nacl.sealedbox_decrypt pEXHQM6cuaF7A= - salt-call --local nacl.sealedbox_decrypt data='pEXHQM6cuaF7A=' sk_file=/etc/salt/pki/master/nacl - salt-call --local nacl.sealedbox_decrypt data='pEXHQM6cuaF7A=' sk='YmFkcGFzcwo=' + salt-run nacl.sealedbox_decrypt pEXHQM6cuaF7A= + salt-run nacl.sealedbox_decrypt data='pEXHQM6cuaF7A=' sk_file=/etc/salt/pki/master/nacl + salt-run nacl.sealedbox_decrypt data='pEXHQM6cuaF7A=' sk='YmFkcGFzcwo=' ''' if data is None: return None + + # ensure data is bytes + data = salt.utils.stringutils.to_bytes(data) + sk = _get_sk(**kwargs) keypair = libnacl.public.SecretKey(sk) b = libnacl.sealed.SealedBox(keypair) @@ -449,9 +469,12 @@ def secretbox_encrypt(data, **kwargs): .. code-block:: bash salt-run nacl.secretbox_encrypt datatoenc - salt-call --local nacl.secretbox_encrypt datatoenc sk_file=/etc/salt/pki/master/nacl - salt-call --local nacl.secretbox_encrypt datatoenc sk='YmFkcGFzcwo=' + salt-run nacl.secretbox_encrypt datatoenc sk_file=/etc/salt/pki/master/nacl + salt-run nacl.secretbox_encrypt datatoenc sk='YmFkcGFzcwo=' ''' + # ensure data is bytes + data = salt.utils.stringutils.to_bytes(data) + sk = _get_sk(**kwargs) b = libnacl.secret.SecretBox(sk) return base64.b64encode(b.encrypt(data)) @@ -466,12 +489,16 @@ def secretbox_decrypt(data, **kwargs): .. code-block:: bash - salt-call nacl.secretbox_decrypt pEXHQM6cuaF7A= - salt-call --local nacl.secretbox_decrypt data='pEXHQM6cuaF7A=' sk_file=/etc/salt/pki/master/nacl - salt-call --local nacl.secretbox_decrypt data='pEXHQM6cuaF7A=' sk='YmFkcGFzcwo=' + salt-run nacl.secretbox_decrypt pEXHQM6cuaF7A= + salt-run nacl.secretbox_decrypt data='pEXHQM6cuaF7A=' sk_file=/etc/salt/pki/master/nacl + salt-run nacl.secretbox_decrypt data='pEXHQM6cuaF7A=' sk='YmFkcGFzcwo=' ''' if data is None: return None + + # ensure data is bytes + data = salt.utils.stringutils.to_bytes(data) + key = _get_sk(**kwargs) b = libnacl.secret.SecretBox(key=key) return b.decrypt(base64.b64decode(data)) diff --git a/salt/sdb/vault.py b/salt/sdb/vault.py index 6ef616b287..598054333b 100644 --- a/salt/sdb/vault.py +++ b/salt/sdb/vault.py @@ -27,7 +27,7 @@ Once configured you can access data using a URL such as: .. code-block:: yaml - password: sdb://myvault/secret/passwords?mypassword + password: sdb://myvault/secret/passwords/mypassword In this URL, ``myvault`` refers to the configuration profile, ``secret/passwords`` is the path where the data resides, and ``mypassword`` is @@ -56,9 +56,17 @@ def set_(key, value, profile=None): ''' Set a key/value pair in the vault service ''' - comps = key.split('?') - path = comps[0] - key = comps[1] + if '?' in key: + __utils__['versions.warn_until']( + 'Neon', + ( + 'Using ? to seperate between the path and key for vault has been deprecated ' + 'and will be removed in {version}. Please just use a /.' + ), + ) + path, key = key.split('?') + else: + path, key = key.rsplit('/', 1) try: url = 'v1/{0}'.format(path) @@ -81,9 +89,17 @@ def get(key, profile=None): ''' Get a value from the vault service ''' - comps = key.split('?') - path = comps[0] - key = comps[1] + if '?' in key: + __utils__['versions.warn_until']( + 'Neon', + ( + 'Using ? to seperate between the path and key for vault has been deprecated ' + 'and will be removed in {version}. Please just use a /.' + ), + ) + path, key = key.split('?') + else: + path, key = key.rsplit('/', 1) try: url = 'v1/{0}'.format(path) diff --git a/salt/state.py b/salt/state.py index 45511d6039..3d368b2c0f 100644 --- a/salt/state.py +++ b/salt/state.py @@ -720,8 +720,12 @@ class State(object): except AttributeError: pillar_enc = six.text_type(pillar_enc).lower() self._pillar_enc = pillar_enc - if initial_pillar: + if initial_pillar and not self._pillar_override: self.opts['pillar'] = initial_pillar + else: + # Compile pillar data + self.opts['pillar'] = self._gather_pillar() + # Reapply overrides on top of compiled pillar if self._pillar_override: self.opts['pillar'] = salt.utils.dictupdate.merge( self.opts['pillar'], @@ -729,8 +733,6 @@ class State(object): self.opts.get('pillar_source_merging_strategy', 'smart'), self.opts.get('renderer', 'yaml'), self.opts.get('pillar_merge_lists', False)) - else: - self.opts['pillar'] = self._gather_pillar() self.state_con = context or {} self.load_modules() self.active = set() @@ -3919,24 +3921,23 @@ class BaseHighState(object): return err if not high: return ret - cumask = os.umask(0o77) - try: - if salt.utils.platform.is_windows(): - # Make sure cache file isn't read-only - self.state.functions['cmd.run']( - ['attrib', '-R', cfn], - python_shell=False, - output_loglevel='quiet') - with salt.utils.files.fopen(cfn, 'w+b') as fp_: - try: - self.serial.dump(high, fp_) - except TypeError: - # Can't serialize pydsl - pass - except (IOError, OSError): - log.error('Unable to write to "state.highstate" cache file %s', cfn) + with salt.utils.files.set_umask(0o077): + try: + if salt.utils.platform.is_windows(): + # Make sure cache file isn't read-only + self.state.functions['cmd.run']( + ['attrib', '-R', cfn], + python_shell=False, + output_loglevel='quiet') + with salt.utils.files.fopen(cfn, 'w+b') as fp_: + try: + self.serial.dump(high, fp_) + except TypeError: + # Can't serialize pydsl + pass + except (IOError, OSError): + log.error('Unable to write to "state.highstate" cache file %s', cfn) - os.umask(cumask) return self.state.call_high(high, orchestration_jid) def compile_highstate(self): diff --git a/salt/states/boto_asg.py b/salt/states/boto_asg.py index 13bd2c3b2d..3530f80bbe 100644 --- a/salt/states/boto_asg.py +++ b/salt/states/boto_asg.py @@ -487,9 +487,10 @@ def present( iargs = {'ami_name': image_name, 'region': region, 'key': key, 'keyid': keyid, 'profile': profile} image_ids = __salt__['boto_ec2.find_images'](**iargs) - if len(image_ids): + if image_ids: # find_images() returns False on failure launch_config[index]['image_id'] = image_ids[0] else: + log.warning("Couldn't find AMI named `%s`, passing literally.", image_name) launch_config[index]['image_id'] = image_name del launch_config[index]['image_name'] break diff --git a/salt/states/cmd.py b/salt/states/cmd.py index 2ae3bbaf7c..2a93458ffb 100644 --- a/salt/states/cmd.py +++ b/salt/states/cmd.py @@ -199,8 +199,7 @@ executed when the state it is watching changes. Example: ``cmd.wait`` itself does not do anything; all functionality is inside its ``mod_watch`` function, which is called by ``watch`` on changes. -``cmd.wait`` will be deprecated in future due to the confusion it causes. The -preferred format is using the :ref:`onchanges Requisite <requisites-onchanges>`, which +The preferred format is using the :ref:`onchanges Requisite <requisites-onchanges>`, which works on ``cmd.run`` as well as on any other state. The example would then look as follows: .. code-block:: yaml diff --git a/salt/states/docker.py b/salt/states/docker.py deleted file mode 100644 index f143f3fedf..0000000000 --- a/salt/states/docker.py +++ /dev/null @@ -1,278 +0,0 @@ -# -*- coding: utf-8 -*- -''' -States to manage Docker containers, images, volumes, and networks - -.. versionchanged:: 2017.7.0 - The legacy Docker state and execution module have been removed, and the - new modules (formerly called ``dockerng`` have taken their places). - -.. important:: - As of the 2017.7.0 release, the states in this module have been separated - into the following four state modules: - - - :mod:`docker_container <salt.states.docker_container>` - States to manage - Docker containers - - :mod:`docker_image <salt.states.docker_image>` - States to manage Docker - images - - :mod:`docker_volume <salt.states.docker_volume>` - States to manage - Docker volumes - - :mod:`docker_network <salt.states.docker_network>` - States to manage - Docker networks - - The reason for this change was to make states and requisites more clear. - For example, imagine this SLS: - - .. code-block:: yaml - - myuser/appimage: - docker.image_present: - - sls: docker.images.appimage - - myapp: - docker.running: - - image: myuser/appimage - - require: - - docker: myuser/appimage - - The new syntax would be: - - .. code-block:: yaml - - myuser/appimage: - docker_image.present: - - sls: docker.images.appimage - - myapp: - docker_container.running: - - image: myuser/appimage - - require: - - docker_image: myuser/appimage - - This is similar to how Salt handles MySQL, MongoDB, Zabbix, and other cases - where the same execution module is used to manage several different kinds - of objects (users, databases, roles, etc.). - - The old syntax will continue to work until the **Fluorine** release of - Salt. -''' -from __future__ import absolute_import, print_function, unicode_literals -import copy -import logging - -# Import salt libs -import salt.utils.args -import salt.utils.versions - -# Enable proper logging -log = logging.getLogger(__name__) # pylint: disable=invalid-name - -# Define the module's virtual name -__virtualname__ = 'docker' -__virtual_aliases__ = ('dockerng', 'moby') - - -def __virtual__(): - ''' - Only load if the docker execution module is available - ''' - if 'docker.version' in __salt__: - return __virtualname__ - return (False, __salt__.missing_fun_string('docker.version')) - - -def running(name, **kwargs): - ''' - .. deprecated:: 2017.7.0 - This state has been moved to :py:func:`docker_container.running - <salt.states.docker_container.running>`. - ''' - ret = __states__['docker_container.running']( - name, - **salt.utils.args.clean_kwargs(**kwargs) - ) - msg = ( - 'The docker.running state has been renamed to ' - 'docker_container.running. To get rid of this warning, update your ' - 'SLS to use the new name.' - ) - salt.utils.versions.warn_until('Fluorine', msg) - ret.setdefault('warnings', []).append(msg) - return ret - - -def stopped(**kwargs): - ''' - .. deprecated:: 2017.7.0 - This state has been moved to :py:func:`docker_container.stopped - <salt.states.docker_container.stopped>`. - ''' - ret = __states__['docker_container.stopped']( - **salt.utils.args.clean_kwargs(**kwargs) - ) - msg = ( - 'The docker.stopped state has been renamed to ' - 'docker_container.stopped. To get rid of this warning, update your ' - 'SLS to use the new name.' - ) - salt.utils.versions.warn_until('Fluorine', msg) - ret.setdefault('warnings', []).append(msg) - return ret - - -def absent(name, **kwargs): - ''' - .. deprecated:: 2017.7.0 - This state has been moved to :py:func:`docker_container.absent - <salt.states.docker_container.absent>`. - ''' - ret = __states__['docker_container.absent']( - name, - **salt.utils.args.clean_kwargs(**kwargs) - ) - msg = ( - 'The docker.absent state has been renamed to ' - 'docker_container.absent. To get rid of this warning, update your ' - 'SLS to use the new name.' - ) - salt.utils.versions.warn_until('Fluorine', msg) - ret.setdefault('warnings', []).append(msg) - return ret - - -def network_present(name, **kwargs): - ''' - .. deprecated:: 2017.7.0 - This state has been moved to :py:func:`docker_network.present - <salt.states.docker_network.present>`. - ''' - ret = __states__['docker_network.present']( - name, - **salt.utils.args.clean_kwargs(**kwargs) - ) - msg = ( - 'The docker.network_present state has been renamed to ' - 'docker_network.present. To get rid of this warning, update your SLS ' - 'to use the new name.' - ) - salt.utils.versions.warn_until('Fluorine', msg) - ret.setdefault('warnings', []).append(msg) - return ret - - -def network_absent(name, **kwargs): - ''' - .. deprecated:: 2017.7.0 - This state has been moved to :py:func:`docker_network.absent - <salt.states.docker_network.absent>`. - ''' - ret = __states__['docker_network.absent']( - name, - **salt.utils.args.clean_kwargs(**kwargs) - ) - msg = ( - 'The docker.network_absent state has been renamed to ' - 'docker_network.absent. To get rid of this warning, update your SLS ' - 'to use the new name.' - ) - salt.utils.versions.warn_until('Fluorine', msg) - ret.setdefault('warnings', []).append(msg) - return ret - - -def image_present(name, **kwargs): - ''' - .. deprecated:: 2017.7.0 - This state has been moved to :py:func:`docker_image.present - <salt.states.docker_image.present>`. - ''' - ret = __states__['docker_image.present']( - name, - **salt.utils.args.clean_kwargs(**kwargs) - ) - msg = ( - 'The docker.image_present state has been renamed to ' - 'docker_image.present. To get rid of this warning, update your SLS ' - 'to use the new name.' - ) - salt.utils.versions.warn_until('Fluorine', msg) - ret.setdefault('warnings', []).append(msg) - return ret - - -def image_absent(**kwargs): - ''' - .. deprecated:: 2017.7.0 - This state has been moved to :py:func:`docker_image.absent - <salt.states.docker_image.absent>`. - ''' - ret = __states__['docker_image.absent']( - **salt.utils.args.clean_kwargs(**kwargs) - ) - msg = ( - 'The docker.image_absent state has been renamed to ' - 'docker_image.absent. To get rid of this warning, update your SLS to ' - 'use the new name.' - ) - salt.utils.versions.warn_until('Fluorine', msg) - ret.setdefault('warnings', []).append(msg) - return ret - - -def volume_present(name, driver=None, driver_opts=None, force=False): - ''' - .. deprecated:: 2017.7.0 - This state has been moved to :py:func:`docker_volume.present - <salt.states.docker_volume.present>`. - ''' - ret = __states__['docker_volume.present'](name, - driver=driver, - driver_opts=driver_opts, - force=force) - msg = ( - 'The docker.volume_present state has been renamed to ' - 'docker_volume.present. To get rid of this warning, update your SLS ' - 'to use the new name.' - ) - salt.utils.versions.warn_until('Fluorine', msg) - ret.setdefault('warnings', []).append(msg) - return ret - - -def volume_absent(name, driver=None): - ''' - .. deprecated:: 2017.7.0 - This state has been moved to :py:func:`docker_volume.absent - <salt.states.docker_volume.absent>`. - ''' - ret = __states__['docker_volume.absent'](name, driver=driver) - msg = ( - 'The docker.volume_absent state has been renamed to ' - 'docker_volume.absent. To get rid of this warning, update your SLS ' - 'to use the new name.' - ) - salt.utils.versions.warn_until('Fluorine', msg) - ret.setdefault('warnings', []).append(msg) - return ret - - -# Handle requisites -def mod_watch(name, sfun=None, **kwargs): - if sfun == 'running': - watch_kwargs = copy.deepcopy(kwargs) - if watch_kwargs.get('watch_action', 'force') == 'force': - watch_kwargs['force'] = True - else: - watch_kwargs['send_signal'] = True - watch_kwargs['force'] = False - return running(name, **watch_kwargs) - - if sfun == 'image_present': - # Force image to be updated - kwargs['force'] = True - return image_present(name, **kwargs) - - return {'name': name, - 'changes': {}, - 'result': False, - 'comment': ('watch requisite is not' - ' implemented for {0}'.format(sfun))} diff --git a/salt/states/docker_network.py b/salt/states/docker_network.py index 4457e6d0e6..6742c00b73 100644 --- a/salt/states/docker_network.py +++ b/salt/states/docker_network.py @@ -890,7 +890,7 @@ def present(name, return ret -def absent(name, driver=None): +def absent(name): ''' Ensure that a network is absent. @@ -909,12 +909,6 @@ def absent(name, driver=None): 'result': False, 'comment': ''} - if driver is not None: - ret.setdefault('warnings', []).append( - 'The \'driver\' argument has no function and will be removed in ' - 'the Fluorine release.' - ) - try: network = __salt__['docker.inspect_network'](name) except CommandExecutionError as exc: diff --git a/salt/states/git.py b/salt/states/git.py index 4bc49df299..63c987d35d 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -2177,10 +2177,6 @@ def detached(name, If a branch or tag is specified it will be resolved to a commit ID and checked out. - ref - .. deprecated:: 2017.7.0 - Use ``rev`` instead. - target Name of the target directory where repository is about to be cloned. @@ -2271,7 +2267,6 @@ def detached(name, ret = {'name': name, 'result': True, 'comment': '', 'changes': {}} - ref = kwargs.pop('ref', None) kwargs = salt.utils.args.clean_kwargs(**kwargs) if kwargs: return _fail( @@ -2279,15 +2274,6 @@ def detached(name, salt.utils.args.invalid_kwargs(kwargs, raise_exc=False) ) - if ref is not None: - rev = ref - deprecation_msg = ( - 'The \'ref\' argument has been renamed to \'rev\' for ' - 'consistency. Please update your SLS to reflect this.' - ) - ret.setdefault('warnings', []).append(deprecation_msg) - salt.utils.versions.warn_until('Fluorine', deprecation_msg) - if not rev: return _fail( ret, @@ -2576,7 +2562,7 @@ def detached(name, 'refs'.format(remote) ) - #get refs and checkout + # get refs and checkout checkout_commit_id = '' if remote_rev_type is 'hash': if __salt__['git.describe']( diff --git a/salt/states/hipchat.py b/salt/states/hipchat.py index 53f870f4da..cb31d58dd9 100644 --- a/salt/states/hipchat.py +++ b/salt/states/hipchat.py @@ -63,7 +63,7 @@ def send_message(name, - api_url: https://hipchat.myteam.com - api_key: peWcBiMOS9HrZG15peWcBiMOS9HrZG15 - api_version: v1 - - color: green + - message_color: green - notify: True The following parameters are required: @@ -96,7 +96,7 @@ def send_message(name, The api version for Hipchat to use, if not specified in the configuration options of master or minion. - color + message_color The color the Hipchat message should be displayed in. One of the following, default: yellow "yellow", "red", "green", "purple", "gray", or "random". diff --git a/salt/states/mount.py b/salt/states/mount.py index 555dd3619c..fead20b7d6 100644 --- a/salt/states/mount.py +++ b/salt/states/mount.py @@ -341,10 +341,12 @@ def mounted(name, mount_invisible_keys = [ 'actimeo', 'comment', + 'credentials', 'direct-io-mode', 'password', - 'retry', 'port', + 'retry', + 'secretfile', ] if extra_mount_invisible_keys: diff --git a/salt/states/pip_state.py b/salt/states/pip_state.py index 1c96884adc..46b6c3de65 100644 --- a/salt/states/pip_state.py +++ b/salt/states/pip_state.py @@ -180,8 +180,8 @@ def _check_pkg_version_format(pkg): def _check_if_installed(prefix, state_pkg_name, version_spec, ignore_installed, force_reinstall, - upgrade, user, cwd, bin_env, index_url): - + upgrade, user, cwd, bin_env, env_vars, + index_url): # result: None means the command failed to run # result: True means the package is installed # result: False means the package is not installed @@ -190,7 +190,8 @@ def _check_if_installed(prefix, state_pkg_name, version_spec, # Check if the requested package is already installed. try: pip_list = __salt__['pip.list'](prefix, bin_env=bin_env, - user=user, cwd=cwd) + user=user, cwd=cwd, + env_vars=env_vars) prefix_realname = _find_key(prefix, pip_list) except (CommandNotFoundError, CommandExecutionError) as err: ret['result'] = None @@ -682,7 +683,8 @@ def installed(name, version_spec = version_spec out = _check_if_installed(prefix, state_pkg_name, version_spec, ignore_installed, force_reinstall, - upgrade, user, cwd, bin_env, index_url) + upgrade, user, cwd, bin_env, env_vars, + index_url) # If _check_if_installed result is None, something went wrong with # the command running. This way we keep stateful output. if out['result'] is None: @@ -823,7 +825,8 @@ def installed(name, # Case for packages that are not an URL if prefix: pipsearch = __salt__['pip.list'](prefix, bin_env, - user=user, cwd=cwd) + user=user, cwd=cwd, + env_vars=env_vars) # If we didnt find the package in the system after # installing it report it diff --git a/salt/states/user.py b/salt/states/user.py index 691300aa5b..f4ae81dd31 100644 --- a/salt/states/user.py +++ b/salt/states/user.py @@ -33,6 +33,7 @@ import salt.utils.dateutils import salt.utils.platform import salt.utils.user from salt.utils.locales import sdecode, sdecode_if_string +from salt.exceptions import CommandExecutionError # Import 3rd-party libs from salt.ext.six import string_types, iteritems @@ -77,7 +78,9 @@ def _changes(name, win_homedrive=None, win_profile=None, win_logonscript=None, - win_description=None): + win_description=None, + allow_uid_change=False, + allow_gid_change=False): ''' Return a dict of the changes required for a user if the user is present, otherwise return False. @@ -191,6 +194,25 @@ def _changes(name, if __salt__['user.get_loginclass'](name) != loginclass: change['loginclass'] = loginclass + errors = [] + if not allow_uid_change and 'uid' in change: + errors.append( + 'Changing uid ({0} -> {1}) not permitted, set allow_uid_change to ' + 'True to force this change. Note that this will not change file ' + 'ownership.'.format(lusr['uid'], uid) + ) + if not allow_gid_change and 'gid' in change: + errors.append( + 'Changing gid ({0} -> {1}) not permitted, set allow_gid_change to ' + 'True to force this change. Note that this will not change file ' + 'ownership.'.format(lusr['gid'], gid) + ) + if errors: + raise CommandExecutionError( + 'Encountered error checking for needed changes', + info=errors + ) + return change @@ -225,7 +247,9 @@ def present(name, win_profile=None, win_logonscript=None, win_description=None, - nologinit=False): + nologinit=False, + allow_uid_change=False, + allow_gid_change=False): ''' Ensure that the named user is present with the specified properties @@ -233,16 +257,28 @@ def present(name, The name of the user to manage uid - The user id to assign, if left empty then the next available user id - will be assigned + The user id to assign. If not specified, and the user does not exist, + then the next available uid will be assigned. gid - The default group id. Also accepts group name. + The id of the default group to assign to the user. Either a group name + or gid can be used. If not specified, and the user does not exist, then + he next available gid will be assigned. - gid_from_name - If True, the default group id will be set to the id of the group with - the same name as the user. If the group does not exist the state will - fail. Default is ``False``. + gid_from_name : False + If ``True``, the default group id will be set to the id of the group + with the same name as the user. If the group does not exist the state + will fail. + + allow_uid_change : False + Set to ``True`` to allow the state to update the uid. + + .. versionadded:: 2018.3.1 + + allow_gid_change : False + Set to ``True`` to allow the state to update the gid. + + .. versionadded:: 2018.3.1 groups A list of groups to assign the user to, pass a list object. If a group @@ -466,33 +502,40 @@ def present(name, ret['result'] = False return ret - changes = _changes(name, - uid, - gid, - groups, - present_optgroups, - remove_groups, - home, - createhome, - password, - enforce_password, - empty_password, - shell, - fullname, - roomnumber, - workphone, - homephone, - loginclass, - date, - mindays, - maxdays, - inactdays, - warndays, - expire, - win_homedrive, - win_profile, - win_logonscript, - win_description) + try: + changes = _changes(name, + uid, + gid, + groups, + present_optgroups, + remove_groups, + home, + createhome, + password, + enforce_password, + empty_password, + shell, + fullname, + roomnumber, + workphone, + homephone, + loginclass, + date, + mindays, + maxdays, + inactdays, + warndays, + expire, + win_homedrive, + win_profile, + win_logonscript, + win_description, + allow_uid_change, + allow_gid_change) + except CommandExecutionError as exc: + ret['result'] = False + ret['comment'] = exc.strerror + return ret if changes: if __opts__['test']: @@ -621,7 +664,13 @@ def present(name, win_homedrive, win_profile, win_logonscript, - win_description) + win_description, + allow_uid_change=True, + allow_gid_change=True) + # allow_uid_change and allow_gid_change passed as True to avoid race + # conditions where a uid/gid is modified outside of Salt. If an + # unauthorized change was requested, it would have been caught the + # first time we ran _changes(). if changes: ret['comment'] = 'These values could not be changed: {0}'.format( diff --git a/salt/states/win_update.py b/salt/states/win_update.py deleted file mode 100644 index 0d9e9cba41..0000000000 --- a/salt/states/win_update.py +++ /dev/null @@ -1,587 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Management of the windows update agent -====================================== - -This module is being deprecated and will be removed in Salt Fluorine. Please use -the ``win_wua`` state module instead. - -.. versionadded:: 2014.7.0 - -Set windows updates to run by category. Default behavior is to install -all updates that do not require user interaction to complete. - -Optionally set ``category`` to a category of your choice to only -install certain updates. Default is to set to install all available updates. - -The following example will install all Security and Critical Updates, -and download but not install standard updates. - -.. code-block:: yaml - - updates: - win_update.installed: - - categories: - - 'Critical Updates' - - 'Security Updates' - - skips: - - downloaded - win_update.downloaded: - - categories: - - 'Updates' - - skips: - - downloaded - -You can also specify a number of features about the update to have a -fine grain approach to specific types of updates. These are the following -features/states of updates available for configuring: - -.. code-block:: text - - 'UI' - User interaction required, skipped by default - 'downloaded' - Already downloaded, included by default - 'present' - Present on computer, skipped by default - 'installed' - Already installed, skipped by default - 'reboot' - Reboot required, included by default - 'hidden' - Skip updates that have been hidden, skipped by default - 'software' - Software updates, included by default - 'driver' - driver updates, included by default - -The following example installs all driver updates that don't require a reboot: -.. code-block:: yaml - - gryffindor: - win_update.installed: - - skips: - - driver: True - - software: False - - reboot: False - -To just update your windows machine, add this your sls: - -.. code-block:: yaml - - updates: - win_update.installed -''' - -# Import Python libs -from __future__ import absolute_import, unicode_literals, print_function -import logging - -# Import 3rd-party libs -# pylint: disable=import-error -from salt.ext import six -from salt.ext.six.moves import range # pylint: disable=redefined-builtin -try: - import win32com.client - import pythoncom - HAS_DEPENDENCIES = True -except ImportError: - HAS_DEPENDENCIES = False -# pylint: enable=import-error - -# Import Salt libs -import salt.utils.platform -import salt.utils.versions - -log = logging.getLogger(__name__) - - -def __virtual__(): - ''' - Only works on Windows systems - ''' - if salt.utils.platform.is_windows() and HAS_DEPENDENCIES: - return True - return False - - -def _gather_update_categories(updateCollection): - ''' - this is a convenience method to gather what categories of updates are available in any update - collection it is passed. Typically though, the download_collection. - Some known categories: - Updates - Windows 7 - Critical Updates - Security Updates - Update Rollups - ''' - categories = [] - for i in range(updateCollection.Count): - update = updateCollection.Item(i) - for j in range(update.Categories.Count): - name = update.Categories.Item(j).Name - if name not in categories: - log.debug('found category: {0}'.format(name)) - categories.append(name) - return categories - - -class PyWinUpdater(object): - def __init__(self, categories=None, skipUI=True, skipDownloaded=False, - skipInstalled=True, skipReboot=False, skipPresent=False, - skipSoftwareUpdates=False, skipDriverUpdates=False, skipHidden=True): - log.debug('CoInitializing the pycom system') - pythoncom.CoInitialize() - - # pylint: disable=invalid-name - self.skipUI = skipUI - self.skipDownloaded = skipDownloaded - self.skipInstalled = skipInstalled - self.skipReboot = skipReboot - self.skipPresent = skipPresent - self.skipHidden = skipHidden - - self.skipSoftwareUpdates = skipSoftwareUpdates - self.skipDriverUpdates = skipDriverUpdates - self.categories = categories - self.foundCategories = None - # pylint: enable=invalid-name - - log.debug('dispatching update_session to keep the session object.') - self.update_session = win32com.client.Dispatch('Microsoft.Update.Session') - - log.debug('update_session got. Now creating a win_searcher to seek out the updates') - self.win_searcher = self.update_session.CreateUpdateSearcher() - - # list of updates that are applicable by current settings. - self.download_collection = win32com.client.Dispatch('Microsoft.Update.UpdateColl') - - # list of updates to be installed. - self.install_collection = win32com.client.Dispatch('Microsoft.Update.UpdateColl') - - # the object responsible for fetching the actual downloads. - self.win_downloader = self.update_session.CreateUpdateDownloader() - self.win_downloader.Updates = self.download_collection - - # the object responsible for the installing of the updates. - self.win_installer = self.update_session.CreateUpdateInstaller() - self.win_installer.Updates = self.install_collection - - # the results of the download process - self.download_results = None - - # the results of the installation process - self.install_results = None - - def Search(self, searchString): - try: - log.debug('beginning search of the passed string: %s', - searchString) - self.search_results = self.win_searcher.Search(searchString) - log.debug('search completed successfully.') - except Exception as exc: - log.info('search for updates failed. %s', exc) - return exc - - log.debug('parsing results. %s updates were found.', - self.search_results.Updates.Count) - try: - for update in self.search_results.Updates: - if update.InstallationBehavior.CanRequestUserInput: - log.debug('Skipped update %s', update.title) - continue - for category in update.Categories: - if self.skipDownloaded and update.IsDownloaded: - continue - if self.categories is None or category.Name in self.categories: - self.download_collection.Add(update) - log.debug('added update %s', update.title) - self.foundCategories = _gather_update_categories(self.download_collection) - return True - except Exception as exc: - log.info('parsing updates failed. %s', exc) - return exc - - def AutoSearch(self): - search_string = '' - searchParams = [] - if self.skipInstalled: - searchParams.append('IsInstalled=0') - else: - searchParams.append('IsInstalled=1') - - if self.skipHidden: - searchParams.append('IsHidden=0') - else: - searchParams.append('IsHidden=1') - - if self.skipReboot: - searchParams.append('RebootRequired=0') - else: - searchParams.append('RebootRequired=1') - - if self.skipPresent: - searchParams.append('IsPresent=0') - else: - searchParams.append('IsPresent=1') - - if len(searchParams) > 1: - for i in searchParams: - search_string += '{0} and '.format(i) - else: - search_string += '{0} and '.format(searchParams[1]) - - if not self.skipSoftwareUpdates and not self.skipDriverUpdates: - search_string += 'Type=\'Software\' or Type=\'Driver\'' - elif not self.skipSoftwareUpdates: - search_string += 'Type=\'Software\'' - elif not self.skipDriverUpdates: - search_string += 'Type=\'Driver\'' - else: - return False - # if there is no type, the is nothing to search. - log.debug('generated search string: %s', search_string) - return self.Search(search_string) - - def Download(self): - try: - if self.download_collection.Count != 0: - self.download_results = self.win_downloader.Download() - else: - log.debug('Skipped downloading, all updates were already cached.') - return True - except Exception as exc: - log.debug('failed in the downloading %s.', exc) - return exc - - def Install(self): - try: - for update in self.search_results.Updates: - if update.IsDownloaded: - self.install_collection.Add(update) - log.debug('Updates prepared. beginning installation') - except Exception as exc: - log.info('Preparing install list failed: %s', exc) - return exc - - # accept eula if not accepted - try: - for update in self.search_results.Updates: - if not update.EulaAccepted: - log.debug('Accepting EULA: %s', update.Title) - update.AcceptEula() - except Exception as exc: - log.info('Accepting Eula failed: %s', exc) - return exc - - if self.install_collection.Count != 0: - log.debug('Install list created, about to install') - updates = [] - try: - self.install_results = self.win_installer.Install() - log.info('Installation of updates complete') - return True - except Exception as exc: - log.info('Installation failed: %s', exc) - return exc - else: - log.info('no new updates.') - return True - - def GetInstallationResults(self): - log.debug('bluger has %s updates in it', self.install_collection.Count) - updates = [] - if self.install_collection.Count == 0: - return {} - for i in range(self.install_collection.Count): - updates.append('{0}: {1}'.format( - self.install_results.GetUpdateResult(i).ResultCode, - self.install_collection.Item(i).Title)) - - log.debug('Update results enumerated, now making a list to pass back') - results = {} - for i, update in enumerate(updates): - results['update {0}'.format(i)] = update - - log.debug('Update information complied. returning') - return results - - def GetDownloadResults(self): - updates = [] - for i in range(self.download_collection.Count): - updates.append('{0}: {1}'.format( - self.download_results.GetUpdateResult(i).ResultCode, - self.download_collection.Item(i).Title)) - results = {} - for i, update in enumerate(updates): - results['update {0}'.format(i)] = update - return results - - def SetCategories(self, categories): - self.categories = categories - - def GetCategories(self): - return self.categories - - def GetAvailableCategories(self): - return self.foundCategories - - def SetSkips(self, skips): - if skips: - for i in skips: - value = i[next(six.iterkeys(i))] - skip = next(six.iterkeys(i)) - self.SetSkip(skip, value) - log.debug('was asked to set %s to %s', skip, value) - - def SetSkip(self, skip, state): - if skip == 'UI': - self.skipUI = state - elif skip == 'downloaded': - self.skipDownloaded = state - elif skip == 'installed': - self.skipInstalled = state - elif skip == 'reboot': - self.skipReboot = state - elif skip == 'present': - self.skipPresent = state - elif skip == 'hidden': - self.skipHidden = state - elif skip == 'software': - self.skipSoftwareUpdates = state - elif skip == 'driver': - self.skipDriverUpdates = state - log.debug('new search state: \n\tUI: %s\n\tDownload: %s\n' - '\tInstalled: %s\n\treboot :%s\n\tPresent: %s\n' - '\thidden: %s\n\tsoftware: %s\n\tdriver: %s', - self.skipUI, self.skipDownloaded, self.skipInstalled, - self.skipReboot, self.skipPresent, self.skipHidden, - self.skipSoftwareUpdates, self.skipDriverUpdates) - - -def _search(win_updater, retries=5): - passed = False - clean = True - comment = '' - while not passed: - log.debug('Searching. tries left: %s', retries) - passed = win_updater.AutoSearch() - log.debug('Done searching: %s', passed) - if isinstance(passed, Exception): - clean = False - comment += 'Failed in the seeking/parsing process:\n\t\t{0}\n'.format(passed) - retries -= 1 - if retries: - comment += '{0} tries to go. retrying\n'.format(retries) - passed = False - else: - comment += 'out of retries. this update round failed.\n' - return (comment, True, retries) - passed = False - if clean: - comment += 'Search was done without error.\n' - return (comment, True, retries) - - -def _download(win_updater, retries=5): - passed = False - clean = True - comment = '' - while not passed: - log.debug('Downloading. tries left: %s', retries) - passed = win_updater.Download() - log.debug('Done downloading: %s', passed) - if isinstance(passed, Exception): - clean = False - comment += 'Failed while trying to download updates:\n\t\t{0}\n'.format(passed) - retries -= 1 - if retries: - comment += '{0} tries to go. retrying\n'.format(retries) - passed = False - else: - comment += 'out of retries. this update round failed.\n' - return (comment, False, retries) - if clean: - comment += 'Download was done without error.\n' - return (comment, True, retries) - - -def _install(win_updater, retries=5): - passed = False - clean = True - comment = '' - while not passed: - log.debug('download_collection is this long: %s', - win_updater.install_collection.Count) - log.debug('Installing. tries left: %s', retries) - passed = win_updater.Install() - log.info('Done installing: %s', passed) - if isinstance(passed, Exception): - clean = False - comment += 'Failed while trying to install the updates.\n\t\t{0}\n'.format(passed) - retries -= 1 - if retries: - comment += '{0} tries to go. retrying\n'.format(retries) - passed = False - else: - comment += 'out of retries. this update round failed.\n' - return (comment, False, retries) - if clean: - comment += 'Install was done without error.\n' - return (comment, True, retries) - - -def installed(name, categories=None, skips=None, retries=10): - ''' - Install specified windows updates. - - name: - if categories is left empty, it will be assumed that you are passing the category option - through the name. These are separate because you can only have one name, but can have - multiple categories. - - categories: - the list of categories to be downloaded. These are simply strings in the update's - information, so there is no enumeration of the categories available. Some known categories: - - .. code-block:: text - - Updates - Windows 7 - Critical Updates - Security Updates - Update Rollups - - skips: - a list of features of the updates to cull by. Available features: - - .. code-block:: text - - 'UI' - User interaction required, skipped by default - 'downloaded' - Already downloaded, skipped by default (downloading) - 'present' - Present on computer, included by default (installing) - 'installed' - Already installed, skipped by default - 'reboot' - Reboot required, included by default - 'hidden' - skip those updates that have been hidden. - 'software' - Software updates, included by default - 'driver' - driver updates, skipped by default - - retries - Number of retries to make before giving up. This is total, not per - step. - ''' - - ret = {'name': name, - 'result': True, - 'changes': {}, - 'comment': ''} - deprecation_msg = 'The \'win_update\' module is deprecated, and will be ' \ - 'removed in Salt Fluorine. Please use the \'win_wua\' ' \ - 'module instead.' - salt.utils.versions.warn_until('Fluorine', deprecation_msg) - ret.setdefault('warnings', []).append(deprecation_msg) - if not categories: - categories = [name] - log.debug('categories to search for are: %s', categories) - win_updater = PyWinUpdater() - win_updater.SetCategories(categories) - win_updater.SetSkips(skips) - - # this is where we be seeking the things! yar! - comment, passed, retries = _search(win_updater, retries) - ret['comment'] += comment - if not passed: - ret['result'] = False - return ret - - # this is where we get all the things! i.e. download updates. - comment, passed, retries = _download(win_updater, retries) - ret['comment'] += comment - if not passed: - ret['result'] = False - return ret - - # this is where we put things in their place! - comment, passed, retries = _install(win_updater, retries) - ret['comment'] += comment - if not passed: - ret['result'] = False - return ret - - try: - ret['changes'] = win_updater.GetInstallationResults() - except Exception: - ret['comment'] += 'could not get results, but updates were installed.' - return ret - - -def downloaded(name, categories=None, skips=None, retries=10): - ''' - Cache updates for later install. - - name: - if categories is left empty, it will be assumed that you are passing the category option - through the name. These are separate because you can only have one name, but can have - multiple categories. - - categories: - the list of categories to be downloaded. These are simply strings in the update's - information, so there is no enumeration of the categories available. Some known categories: - - .. code-block:: text - - Updates - Windows 7 - Critical Updates - Security Updates - Update Rollups - - skips: - a list of features of the updates to cull by. Available features: - - .. code-block:: text - - 'UI' - User interaction required, skipped by default - 'downloaded' - Already downloaded, skipped by default (downloading) - 'present' - Present on computer, included by default (installing) - 'installed' - Already installed, skipped by default - 'reboot' - Reboot required, included by default - 'hidden' - skip those updates that have been hidden. - 'software' - Software updates, included by default - 'driver' - driver updates, skipped by default - - retries - Number of retries to make before giving up. This is total, not per - step. - ''' - ret = {'name': name, - 'result': True, - 'changes': {}, - 'comment': ''} - - deprecation_msg = 'The \'win_update\' module is deprecated, and will be ' \ - 'removed in Salt Fluorine. Please use the \'win_wua\' ' \ - 'module instead.' - salt.utils.versions.warn_until('Fluorine', deprecation_msg) - ret.setdefault('warnings', []).append(deprecation_msg) - - if not categories: - categories = [name] - log.debug('categories to search for are: %s', categories) - win_updater = PyWinUpdater() - win_updater.SetCategories(categories) - win_updater.SetSkips(skips) - - # this is where we be seeking the things! yar! - comment, passed, retries = _search(win_updater, retries) - ret['comment'] += comment - if not passed: - ret['result'] = False - return ret - - # this is where we get all the things! i.e. download updates. - comment, passed, retries = _download(win_updater, retries) - ret['comment'] += comment - if not passed: - ret['result'] = False - return ret - - try: - ret['changes'] = win_updater.GetDownloadResults() - except Exception: - ret['comment'] += 'could not get results, but updates were downloaded.' - - return ret diff --git a/salt/states/zabbix_host.py b/salt/states/zabbix_host.py index 65f8303162..ed54b845a1 100644 --- a/salt/states/zabbix_host.py +++ b/salt/states/zabbix_host.py @@ -193,7 +193,7 @@ def present(host, groups, interfaces, **kwargs): host_exists = __salt__['zabbix.host_exists'](host, **connection_args) if host_exists: - host = __salt__['zabbix.host_get'](name=host, **connection_args)[0] + host = __salt__['zabbix.host_get'](host=host, **connection_args)[0] hostid = host['hostid'] update_proxy = False @@ -457,7 +457,7 @@ def assign_templates(host, templates, **kwargs): ret['comment'] = comment_host_templ_notupdated return ret - host_info = __salt__['zabbix.host_get'](name=host, **connection_args)[0] + host_info = __salt__['zabbix.host_get'](host=host, **connection_args)[0] hostid = host_info['hostid'] if not templates: diff --git a/salt/transport/tcp.py b/salt/transport/tcp.py index 334ed0a3ad..b664a2a7ad 100644 --- a/salt/transport/tcp.py +++ b/salt/transport/tcp.py @@ -21,6 +21,7 @@ import errno import salt.crypt import salt.utils.async import salt.utils.event +import salt.utils.files import salt.utils.platform import salt.utils.process import salt.utils.verify @@ -1386,11 +1387,8 @@ class TCPPubServerChannel(salt.transport.server.PubServerChannel): # Securely create socket log.info('Starting the Salt Puller on %s', pull_uri) - old_umask = os.umask(0o177) - try: + with salt.utils.files.set_umask(0o177): pull_sock.start() - finally: - os.umask(old_umask) # run forever try: diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py index ad3dbc27e9..89c0292bde 100644 --- a/salt/transport/zeromq.py +++ b/salt/transport/zeromq.py @@ -19,6 +19,7 @@ from random import randint import salt.auth import salt.crypt import salt.utils.event +import salt.utils.files import salt.utils.minions import salt.utils.process import salt.utils.stringutils @@ -806,11 +807,8 @@ class ZeroMQPubServerChannel(salt.transport.server.PubServerChannel): # Securely create socket log.info('Starting the Salt Puller on %s', pull_uri) - old_umask = os.umask(0o177) - try: + with salt.utils.files.set_umask(0o177): pull_sock.bind(pull_uri) - finally: - os.umask(old_umask) try: while True: diff --git a/salt/utils/args.py b/salt/utils/args.py index f2c9a10d51..91e71193f8 100644 --- a/salt/utils/args.py +++ b/salt/utils/args.py @@ -142,7 +142,8 @@ def yamlify_arg(arg): return arg if arg.strip() == '': - # Because YAML loads empty strings as None, we return the original string + # Because YAML loads empty (or all whitespace) strings as None, we + # return the original string # >>> import yaml # >>> yaml.load('') is None # True @@ -151,6 +152,9 @@ def yamlify_arg(arg): return arg elif '_' in arg and all([x in '0123456789_' for x in arg.strip()]): + # When the stripped string includes just digits and underscores, the + # underscores are ignored and the digits are combined together and + # loaded as an int. We don't want that, so return the original value. return arg try: @@ -177,6 +181,14 @@ def yamlify_arg(arg): else: return arg + elif isinstance(arg, list): + # lists must be wrapped in brackets + if (isinstance(original_arg, six.string_types) and + not original_arg.startswith('[')): + return original_arg + else: + return arg + elif arg is None \ or isinstance(arg, (list, float, six.integer_types, six.string_types)): # yaml.safe_load will load '|' as '', don't let it do that. diff --git a/salt/utils/docker/__init__.py b/salt/utils/docker/__init__.py index 4edaca7860..268992389d 100644 --- a/salt/utils/docker/__init__.py +++ b/salt/utils/docker/__init__.py @@ -10,7 +10,6 @@ input as formatted by states. from __future__ import absolute_import, print_function, unicode_literals import copy import logging -import os # Import Salt libs import salt.utils.args diff --git a/salt/utils/event.py b/salt/utils/event.py index 4681bccb04..29cfd7ea7c 100644 --- a/salt/utils/event.py +++ b/salt/utils/event.py @@ -75,6 +75,7 @@ import salt.payload import salt.utils.async import salt.utils.cache import salt.utils.dicttrim +import salt.utils.files import salt.utils.platform import salt.utils.process import salt.utils.stringutils @@ -1018,12 +1019,9 @@ class AsyncEventPublisher(object): ) log.info('Starting pull socket on {0}'.format(epull_uri)) - old_umask = os.umask(0o177) - try: + with salt.utils.files.set_umask(0o177): self.publisher.start() self.puller.start() - finally: - os.umask(old_umask) def handle_publish(self, package, _): ''' @@ -1106,8 +1104,7 @@ class EventPublisher(salt.utils.process.SignalHandlingMultiprocessingProcess): ) # Start the master event publisher - old_umask = os.umask(0o177) - try: + with salt.utils.files.set_umask(0o177): self.publisher.start() self.puller.start() if (self.opts['ipc_mode'] != 'tcp' and ( @@ -1115,8 +1112,6 @@ class EventPublisher(salt.utils.process.SignalHandlingMultiprocessingProcess): self.opts['external_auth'])): os.chmod(os.path.join( self.opts['sock_dir'], 'master_event_pub.ipc'), 0o666) - finally: - os.umask(old_umask) # Make sure the IO loop and respective sockets are closed and # destroyed diff --git a/salt/utils/extmods.py b/salt/utils/extmods.py index 7114d190d9..18a794fab1 100644 --- a/salt/utils/extmods.py +++ b/salt/utils/extmods.py @@ -11,6 +11,7 @@ import shutil # Import salt libs import salt.fileclient +import salt.utils.files import salt.utils.hashutils import salt.utils.path import salt.utils.url @@ -71,85 +72,83 @@ def sync(opts, remote = set() source = salt.utils.url.create('_' + form) mod_dir = os.path.join(opts['extension_modules'], '{0}'.format(form)) - cumask = os.umask(0o77) touched = False - try: - if not os.path.isdir(mod_dir): - log.info('Creating module dir \'%s\'', mod_dir) - try: - os.makedirs(mod_dir) - except (IOError, OSError): - log.error( - 'Cannot create cache module directory %s. Check ' - 'permissions.', mod_dir - ) - fileclient = salt.fileclient.get_file_client(opts) - for sub_env in saltenv: - log.info( - 'Syncing %s for environment \'%s\'', form, sub_env - ) - cache = [] - log.info( - 'Loading cache from {0}, for {1})'.format(source, sub_env) - ) - # Grab only the desired files (.py, .pyx, .so) - cache.extend( - fileclient.cache_dir( - source, sub_env, include_empty=False, - include_pat=r'E@\.(pyx?|so|zip)$', exclude_pat=None - ) - ) - local_cache_dir = os.path.join( - opts['cachedir'], - 'files', - sub_env, - '_{0}'.format(form) + with salt.utils.files.set_umask(0o077): + try: + if not os.path.isdir(mod_dir): + log.info('Creating module dir \'%s\'', mod_dir) + try: + os.makedirs(mod_dir) + except (IOError, OSError): + log.error( + 'Cannot create cache module directory %s. Check ' + 'permissions.', mod_dir ) - log.debug('Local cache dir: \'%s\'', local_cache_dir) - for fn_ in cache: - relpath = os.path.relpath(fn_, local_cache_dir) - relname = os.path.splitext(relpath)[0].replace(os.sep, '.') - if extmod_whitelist and form in extmod_whitelist and relname not in extmod_whitelist[form]: - continue - if extmod_blacklist and form in extmod_blacklist and relname in extmod_blacklist[form]: - continue - remote.add(relpath) - dest = os.path.join(mod_dir, relpath) - log.info('Copying \'%s\' to \'%s\'', fn_, dest) - if os.path.isfile(dest): - # The file is present, if the sum differs replace it - hash_type = opts.get('hash_type', 'md5') - src_digest = salt.utils.hashutils.get_hash(fn_, hash_type) - dst_digest = salt.utils.hashutils.get_hash(dest, hash_type) - if src_digest != dst_digest: - # The downloaded file differs, replace! + fileclient = salt.fileclient.get_file_client(opts) + for sub_env in saltenv: + log.info( + 'Syncing %s for environment \'%s\'', form, sub_env + ) + cache = [] + log.info( + 'Loading cache from {0}, for {1})'.format(source, sub_env) + ) + # Grab only the desired files (.py, .pyx, .so) + cache.extend( + fileclient.cache_dir( + source, sub_env, include_empty=False, + include_pat=r'E@\.(pyx?|so|zip)$', exclude_pat=None + ) + ) + local_cache_dir = os.path.join( + opts['cachedir'], + 'files', + sub_env, + '_{0}'.format(form) + ) + log.debug('Local cache dir: \'%s\'', local_cache_dir) + for fn_ in cache: + relpath = os.path.relpath(fn_, local_cache_dir) + relname = os.path.splitext(relpath)[0].replace(os.sep, '.') + if extmod_whitelist and form in extmod_whitelist and relname not in extmod_whitelist[form]: + continue + if extmod_blacklist and form in extmod_blacklist and relname in extmod_blacklist[form]: + continue + remote.add(relpath) + dest = os.path.join(mod_dir, relpath) + log.info('Copying \'%s\' to \'%s\'', fn_, dest) + if os.path.isfile(dest): + # The file is present, if the sum differs replace it + hash_type = opts.get('hash_type', 'md5') + src_digest = salt.utils.hashutils.get_hash(fn_, hash_type) + dst_digest = salt.utils.hashutils.get_hash(dest, hash_type) + if src_digest != dst_digest: + # The downloaded file differs, replace! + shutil.copyfile(fn_, dest) + ret.append('{0}.{1}'.format(form, relname)) + else: + dest_dir = os.path.dirname(dest) + if not os.path.isdir(dest_dir): + os.makedirs(dest_dir) shutil.copyfile(fn_, dest) ret.append('{0}.{1}'.format(form, relname)) - else: - dest_dir = os.path.dirname(dest) - if not os.path.isdir(dest_dir): - os.makedirs(dest_dir) - shutil.copyfile(fn_, dest) - ret.append('{0}.{1}'.format(form, relname)) - touched = bool(ret) - if opts['clean_dynamic_modules'] is True: - current = set(_listdir_recursively(mod_dir)) - for fn_ in current - remote: - full = os.path.join(mod_dir, fn_) - if os.path.isfile(full): - touched = True - os.remove(full) - # Cleanup empty dirs - while True: - emptydirs = _list_emptydirs(mod_dir) - if not emptydirs: - break - for emptydir in emptydirs: - touched = True - shutil.rmtree(emptydir, ignore_errors=True) - except Exception as exc: - log.error('Failed to sync %s module: %s', form, exc) - finally: - os.umask(cumask) + touched = bool(ret) + if opts['clean_dynamic_modules'] is True: + current = set(_listdir_recursively(mod_dir)) + for fn_ in current - remote: + full = os.path.join(mod_dir, fn_) + if os.path.isfile(full): + touched = True + os.remove(full) + # Cleanup empty dirs + while True: + emptydirs = _list_emptydirs(mod_dir) + if not emptydirs: + break + for emptydir in emptydirs: + touched = True + shutil.rmtree(emptydir, ignore_errors=True) + except Exception as exc: + log.error('Failed to sync %s module: %s', form, exc) return ret, touched diff --git a/salt/utils/files.py b/salt/utils/files.py index 397b2206fb..eddb53b410 100644 --- a/salt/utils/files.py +++ b/salt/utils/files.py @@ -299,20 +299,29 @@ def wait_lock(path, lock_fn=None, timeout=5, sleep=0.1, time_start=None): log.trace('Write lock for %s (%s) released', path, lock_fn) +def get_umask(): + ''' + Returns the current umask + ''' + ret = os.umask(0) # pylint: disable=blacklisted-function + os.umask(ret) # pylint: disable=blacklisted-function + return ret + + @contextlib.contextmanager def set_umask(mask): ''' Temporarily set the umask and restore once the contextmanager exits ''' - if salt.utils.platform.is_windows(): - # Don't attempt on Windows + if mask is None or salt.utils.platform.is_windows(): + # Don't attempt on Windows, or if no mask was passed yield else: try: - orig_mask = os.umask(mask) + orig_mask = os.umask(mask) # pylint: disable=blacklisted-function yield finally: - os.umask(orig_mask) + os.umask(orig_mask) # pylint: disable=blacklisted-function def fopen(*args, **kwargs): diff --git a/salt/utils/openstack/neutron.py b/salt/utils/openstack/neutron.py index 2a7a65f137..0af2de3140 100644 --- a/salt/utils/openstack/neutron.py +++ b/salt/utils/openstack/neutron.py @@ -81,6 +81,13 @@ class SaltNeutron(NeutronShell): ''' Set up neutron credentials ''' + __utils__['versions.warn_until']( + 'Neon', + ( + 'The neutron module has been deprecated and will be removed in {version}. ' + 'Please update to using the neutronng module' + ), + ) if not HAS_NEUTRON: return None diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py index ffe42bee23..4a4aa1c3a6 100644 --- a/salt/utils/parsers.py +++ b/salt/utils/parsers.py @@ -715,9 +715,8 @@ class LogLevelMixIn(six.with_metaclass(MixInMeta, object)): # verify the default if logfile is not None and not logfile.startswith(('tcp://', 'udp://', 'file://')): # Logfile is not using Syslog, verify - current_umask = os.umask(0o027) - verify_files([logfile], self.config['user']) - os.umask(current_umask) + with salt.utils.files.set_umask(0o027): + verify_files([logfile], self.config['user']) if logfile is None: # Use the default setting if the logfile wasn't explicity set diff --git a/salt/utils/process.py b/salt/utils/process.py index 6dafaf0441..a9ae5138ee 100644 --- a/salt/utils/process.py +++ b/salt/utils/process.py @@ -80,7 +80,7 @@ def daemonize(redirect_out=True): os.chdir('/') # noinspection PyArgumentList os.setsid() - os.umask(18) + os.umask(0o022) # pylint: disable=blacklisted-function # do second fork try: diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index d6eb78225b..c182dabcb1 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -426,23 +426,27 @@ class Schedule(object): # Grab run, assume True run = data.get('run', True) + run_schedule_jobs_in_background = self.opts.get('run_schedule_jobs_in_background', True) if run: - multiprocessing_enabled = self.opts.get('multiprocessing', True) - if multiprocessing_enabled: - thread_cls = salt.utils.process.SignalHandlingMultiprocessingProcess - else: - thread_cls = threading.Thread + if run_schedule_jobs_in_background: + multiprocessing_enabled = self.opts.get('multiprocessing', True) + if multiprocessing_enabled: + thread_cls = salt.utils.process.SignalHandlingMultiprocessingProcess + else: + thread_cls = threading.Thread - if multiprocessing_enabled: - with salt.utils.process.default_signals(signal.SIGINT, signal.SIGTERM): + if multiprocessing_enabled: + with salt.utils.process.default_signals(signal.SIGINT, signal.SIGTERM): + proc = thread_cls(target=self.handle_func, args=(multiprocessing_enabled, func, data)) + # Reset current signals before starting the process in + # order not to inherit the current signal handlers + proc.start() + proc.join() + else: proc = thread_cls(target=self.handle_func, args=(multiprocessing_enabled, func, data)) - # Reset current signals before starting the process in - # order not to inherit the current signal handlers proc.start() - proc.join() else: - proc = thread_cls(target=self.handle_func, args=(multiprocessing_enabled, func, data)) - proc.start() + func(data) def enable_schedule(self): ''' @@ -1474,7 +1478,7 @@ class Schedule(object): # If there is no job specific skip_during_range available, # grab the global which defaults to None. - if 'skip_during_range' not in data: + if 'skip_during_range' not in data and self.skip_during_range: data['skip_during_range'] = self.skip_during_range if 'skip_during_range' in data and data['skip_during_range']: diff --git a/salt/utils/user.py b/salt/utils/user.py index 6a6dc5e478..a6493ae3a1 100644 --- a/salt/utils/user.py +++ b/salt/utils/user.py @@ -254,7 +254,7 @@ def chugid_and_umask(runas, umask, group=None): if set_runas or set_grp: chugid(runas_user, runas_grp) if umask is not None: - os.umask(umask) + os.umask(umask) # pylint: disable=blacklisted-function def get_default_group(user): diff --git a/salt/utils/verify.py b/salt/utils/verify.py index 0eeb95b402..9dd8dd4b84 100644 --- a/salt/utils/verify.py +++ b/salt/utils/verify.py @@ -230,12 +230,11 @@ def verify_env( continue if not os.path.isdir(dir_): try: - cumask = os.umask(18) # 077 - os.makedirs(dir_) + with salt.utils.files.set_umask(0o022): + os.makedirs(dir_) # If starting the process as root, chown the new dirs if os.getuid() == 0: os.chown(dir_, uid, gid) - os.umask(cumask) except OSError as err: msg = 'Failed to create directory path "{0}" - {1}\n' sys.stderr.write(msg.format(dir_, err)) diff --git a/tests/integration/files/conf/cloud.providers.d/profitbricks.conf b/tests/integration/files/conf/cloud.providers.d/profitbricks.conf index 292e2e3230..cb75738d68 100644 --- a/tests/integration/files/conf/cloud.providers.d/profitbricks.conf +++ b/tests/integration/files/conf/cloud.providers.d/profitbricks.conf @@ -1,6 +1,6 @@ profitbricks-config: - username: '' - password: '' + username: 'foo' + password: 'bar' datacenter_id: 74d65326-d9b7-41c3-9f51-73ffe0fcd16d driver: profitbricks ssh_public_key: ~/.ssh/id_rsa.pub diff --git a/tests/integration/files/file/base/issue-46127-pip-env-vars.sls b/tests/integration/files/file/base/issue-46127-pip-env-vars.sls new file mode 100644 index 0000000000..e9e415eb87 --- /dev/null +++ b/tests/integration/files/file/base/issue-46127-pip-env-vars.sls @@ -0,0 +1,38 @@ +{%- set virtualenv_base = salt['runtests_helpers.get_salt_temp_dir_for_path']('virtualenv-12-base-1') -%} +{%- set virtualenv_test = salt['runtests_helpers.get_salt_temp_dir_for_path']('issue-46127-pip-env-vars') -%} + +{{ virtualenv_base }}: + virtualenv.managed: + - system_site_packages: False + - distribute: True + +install_older_venv_1: + pip.installed: + - name: 'virtualenv < 13.0' + - bin_env: {{ virtualenv_base }} + - require: + - virtualenv: {{ virtualenv_base }} + +# For this test we need to make sure that the virtualenv used in the +# 'issue-46127-setup' pip.installed state below was created using +# virtualenv < 13.0. virtualenvs created using later versions make +# packages with custom setuptools prefixes relative to the virtualenv +# itself, which makes the use of env_vars obsolete. +# Thus, the two states above ensure that the 'base' venv has +# a version old enough to exhibit the behavior we want to test. + +setup_test_virtualenv_1: + cmd.run: + - name: {{ virtualenv_base }}/bin/virtualenv {{ virtualenv_test }} + - onchanges: + - pip: install_older_venv_1 + +issue-46127-setup: + pip.installed: + - name: 'carbon < 1.3' + - no_deps: True + - env_vars: + PYTHONPATH: "/opt/graphite/lib/:/opt/graphite/webapp/" + - bin_env: {{ virtualenv_test }} + - require: + - cmd: setup_test_virtualenv_1 diff --git a/tests/integration/modules/test_nacl.py b/tests/integration/modules/test_nacl.py new file mode 100644 index 0000000000..10a2c50572 --- /dev/null +++ b/tests/integration/modules/test_nacl.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +''' +Tests for the salt-run command +''' +# Import Python libs +from __future__ import absolute_import, print_function, unicode_literals + +import salt.utils.stringutils + +# Import Salt Testing libs +from tests.support.case import ModuleCase +from tests.support.unit import skipIf + +try: + import libnacl # pylint: disable=unused-import + HAS_LIBNACL = True +except ImportError: + HAS_LIBNACL = False + + +@skipIf(not HAS_LIBNACL, 'skipping test_nacl, libnacl is unavailable') +class NaclTest(ModuleCase): + ''' + Test the nacl runner + ''' + def test_keygen(self): + ''' + Test keygen + ''' + # Store the data + ret = self.run_function( + 'nacl.keygen', + ) + self.assertIn('pk', ret) + self.assertIn('sk', ret) + + def test_enc_dec(self): + ''' + Generate keys, encrypt, then decrypt. + ''' + # Store the data + ret = self.run_function( + 'nacl.keygen', + ) + self.assertIn('pk', ret) + self.assertIn('sk', ret) + pk = ret['pk'] + sk = ret['sk'] + + unencrypted_data = salt.utils.stringutils.to_bytes('hello') + + # Encrypt with pk + ret = self.run_function( + 'nacl.enc', + data=unencrypted_data, + pk=pk, + ) + encrypted_data = ret + + # Decrypt with sk + ret = self.run_function( + 'nacl.dec', + data=encrypted_data, + sk=sk, + ) + self.assertEqual(unencrypted_data, ret) diff --git a/tests/integration/runners/test_nacl.py b/tests/integration/runners/test_nacl.py new file mode 100644 index 0000000000..4c93da6211 --- /dev/null +++ b/tests/integration/runners/test_nacl.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +''' +Tests for the salt-run command +''' +# Import Python libs +from __future__ import absolute_import, print_function, unicode_literals + +# Import Salt Testing libs +from tests.support.case import ShellCase +from tests.support.unit import skipIf + +try: + import libnacl # pylint: disable=unused-import + HAS_LIBNACL = True +except ImportError: + HAS_LIBNACL = False + + +@skipIf(not HAS_LIBNACL, 'skipping test_nacl, libnacl is unavailable') +class NaclTest(ShellCase): + ''' + Test the nacl runner + ''' + def test_keygen(self): + ''' + Test keygen + ''' + # Store the data + ret = self.run_run_plus( + 'nacl.keygen', + ) + self.assertIn('pk', ret['return']) + self.assertIn('sk', ret['return']) + + def test_enc(self): + ''' + Test keygen + ''' + # Store the data + ret = self.run_run_plus( + 'nacl.keygen', + ) + self.assertIn('pk', ret['return']) + self.assertIn('sk', ret['return']) + pk = ret['return']['pk'] + sk = ret['return']['sk'] + + unencrypted_data = 'hello' + + # Encrypt with pk + ret = self.run_run_plus( + 'nacl.enc', + data=unencrypted_data, + pk=pk, + ) + self.assertIn('return', ret) + + def test_enc_dec(self): + ''' + Store, list, fetch, then flush data + ''' + # Store the data + ret = self.run_run_plus( + 'nacl.keygen', + ) + self.assertIn('pk', ret['return']) + self.assertIn('sk', ret['return']) + pk = ret['return']['pk'] + sk = ret['return']['sk'] + + unencrypted_data = 'hello' + + # Encrypt with pk + ret = self.run_run_plus( + 'nacl.enc', + data=unencrypted_data, + pk=pk, + ) + self.assertIn('return', ret) + encrypted_data = ret['return'] + + # Decrypt with sk + ret = self.run_run_plus( + 'nacl.dec', + data=encrypted_data, + sk=sk, + ) + self.assertIn('return', ret) + self.assertEqual(unencrypted_data, ret['return']) diff --git a/tests/integration/scheduler/test_eval.py b/tests/integration/scheduler/test_eval.py index c16f0495e6..896f9f42f1 100644 --- a/tests/integration/scheduler/test_eval.py +++ b/tests/integration/scheduler/test_eval.py @@ -515,3 +515,32 @@ class SchedulerEvalTest(ModuleCase, SaltReturnAssertsMixin): ret = self.schedule.job_status('job1') self.assertNotIn('_last_run', ret) self.assertEqual(ret['_skip_reason'], 'disabled') + + def test_eval_run_on_start(self): + ''' + verify that scheduled job is run when minion starts + ''' + job = { + 'schedule': { + 'job1': { + 'function': 'test.ping', + 'hours': '1', + 'run_on_start': True + } + } + } + + # Add job to schedule + self.schedule.opts.update(job) + + # eval at 2:00pm, will run. + run_time = dateutil_parser.parse('11/29/2017 2:00pm') + self.schedule.eval(now=run_time) + ret = self.schedule.job_status('job1') + self.assertEqual(ret['_last_run'], run_time) + + # eval at 3:00pm, will run. + run_time = dateutil_parser.parse('11/29/2017 3:00pm') + self.schedule.eval(now=run_time) + ret = self.schedule.job_status('job1') + self.assertEqual(ret['_last_run'], run_time) diff --git a/tests/integration/shell/test_call.py b/tests/integration/shell/test_call.py index 3dff8fea5e..7d51174fd9 100644 --- a/tests/integration/shell/test_call.py +++ b/tests/integration/shell/test_call.py @@ -363,58 +363,56 @@ class CallTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin def test_issue_14979_output_file_permissions(self): output_file = os.path.join(TMP, 'issue-14979') - current_umask = os.umask(0o077) - try: - # Let's create an initial output file with some data - self.run_script( - 'salt-call', - '-c {0} --output-file={1} -g'.format( - self.get_config_dir(), - output_file - ), - catch_stderr=True, - with_retcode=True - ) - stat1 = os.stat(output_file) + with salt.utils.files.set_umask(0o077): + try: + # Let's create an initial output file with some data + self.run_script( + 'salt-call', + '-c {0} --output-file={1} -g'.format( + self.get_config_dir(), + output_file + ), + catch_stderr=True, + with_retcode=True + ) + stat1 = os.stat(output_file) - # Let's change umask - os.umask(0o777) + # Let's change umask + os.umask(0o777) # pylint: disable=blacklisted-function - self.run_script( - 'salt-call', - '-c {0} --output-file={1} --output-file-append -g'.format( - self.get_config_dir(), - output_file - ), - catch_stderr=True, - with_retcode=True - ) - stat2 = os.stat(output_file) - self.assertEqual(stat1.st_mode, stat2.st_mode) - # Data was appeneded to file - self.assertTrue(stat1.st_size < stat2.st_size) + self.run_script( + 'salt-call', + '-c {0} --output-file={1} --output-file-append -g'.format( + self.get_config_dir(), + output_file + ), + catch_stderr=True, + with_retcode=True + ) + stat2 = os.stat(output_file) + self.assertEqual(stat1.st_mode, stat2.st_mode) + # Data was appeneded to file + self.assertTrue(stat1.st_size < stat2.st_size) - # Let's remove the output file - os.unlink(output_file) - - # Not appending data - self.run_script( - 'salt-call', - '-c {0} --output-file={1} -g'.format( - self.get_config_dir(), - output_file - ), - catch_stderr=True, - with_retcode=True - ) - stat3 = os.stat(output_file) - # Mode must have changed since we're creating a new log file - self.assertNotEqual(stat1.st_mode, stat3.st_mode) - finally: - if os.path.exists(output_file): + # Let's remove the output file os.unlink(output_file) - # Restore umask - os.umask(current_umask) + + # Not appending data + self.run_script( + 'salt-call', + '-c {0} --output-file={1} -g'.format( + self.get_config_dir(), + output_file + ), + catch_stderr=True, + with_retcode=True + ) + stat3 = os.stat(output_file) + # Mode must have changed since we're creating a new log file + self.assertNotEqual(stat1.st_mode, stat3.st_mode) + finally: + if os.path.exists(output_file): + os.unlink(output_file) @skipIf(sys.platform.startswith('win'), 'This test does not apply on Win') def test_42116_cli_pillar_override(self): diff --git a/tests/integration/states/test_pip.py b/tests/integration/states/test_pip.py index 0ea7d8cd5f..692067eb7f 100644 --- a/tests/integration/states/test_pip.py +++ b/tests/integration/states/test_pip.py @@ -524,3 +524,78 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin): finally: if os.path.isdir(venv_dir): shutil.rmtree(venv_dir) + + def test_46127_pip_env_vars(self): + ''' + Test that checks if env_vars passed to pip.installed are also passed + to pip.freeze while checking for existing installations + ''' + # This issue is most easily checked while installing carbon + # Much of the code here comes from the test_weird_install function above + ographite = '/opt/graphite' + if os.path.isdir(ographite): + self.skipTest( + 'You already have \'{0}\'. This test would overwrite this ' + 'directory'.format(ographite) + ) + try: + os.makedirs(ographite) + except OSError as err: + if err.errno == errno.EACCES: + # Permission denied + self.skipTest( + 'You don\'t have the required permissions to run this test' + ) + finally: + if os.path.isdir(ographite): + shutil.rmtree(ographite) + + venv_dir = os.path.join(RUNTIME_VARS.TMP, 'issue-46127-pip-env-vars') + try: + # We may be able to remove this, I had to add it because the custom + # modules from the test suite weren't available in the jinja + # context when running the call to state.sls that comes after. + self.run_function('saltutil.sync_modules') + # Since we don't have the virtualenv created, pip.installed will + # thrown and error. + ret = self.run_function( + 'state.sls', mods='issue-46127-pip-env-vars' + ) + self.assertSaltTrueReturn(ret) + for key in six.iterkeys(ret): + self.assertTrue(ret[key]['result']) + if ret[key]['name'] != 'carbon < 1.3': + continue + self.assertEqual( + ret[key]['comment'], + 'All packages were successfully installed' + ) + break + else: + raise Exception('Expected state did not run') + # Run the state again. Now the already installed message should + # appear + ret = self.run_function( + 'state.sls', mods='issue-46127-pip-env-vars' + ) + self.assertSaltTrueReturn(ret) + # We cannot use assertInSaltComment here because we need to skip + # some of the state return parts + for key in six.iterkeys(ret): + self.assertTrue(ret[key]['result']) + # As we are re-running the formula, some states will not be run + # and "name" may or may not be present, so we use .get() pattern + if ret[key].get('name', '') != 'carbon < 1.3': + continue + self.assertEqual( + ret[key]['comment'], + ('Python package carbon < 1.3 was already installed\n' + 'All packages were successfully installed')) + break + else: + raise Exception('Expected state did not run') + finally: + if os.path.isdir(ographite): + shutil.rmtree(ographite) + if os.path.isdir(venv_dir): + shutil.rmtree(venv_dir) diff --git a/tests/unit/modules/test_pip.py b/tests/unit/modules/test_pip.py index 25f8db567f..bb346ba2cc 100644 --- a/tests/unit/modules/test_pip.py +++ b/tests/unit/modules/test_pip.py @@ -937,6 +937,27 @@ class PipTestCase(TestCase, LoaderModuleMockMixin): ) self.assertEqual(ret, eggs) + mock = MagicMock( + return_value={ + 'retcode': 0, + 'stdout': '\n'.join(eggs) + } + ) + # Passing env_vars passes them to underlying command? + with patch.dict(pip.__salt__, {'cmd.run_all': mock}): + with patch('salt.modules.pip.version', + MagicMock(return_value='6.1.1')): + ret = pip.freeze(env_vars={"foo": "bar"}) + mock.assert_called_once_with( + ['pip', 'freeze'], + cwd=None, + runas=None, + use_vt=False, + python_shell=False, + env={"foo": "bar"} + ) + self.assertEqual(ret, eggs) + # Non zero returncode raises exception? mock = MagicMock(return_value={'retcode': 1, 'stderr': 'CABOOOOMMM!'}) with patch.dict(pip.__salt__, {'cmd.run_all': mock}): diff --git a/tests/unit/netapi/rest_tornado/test_utils.py b/tests/unit/netapi/rest_tornado/test_utils.py index 5df66cb2d1..5ba08a31f3 100644 --- a/tests/unit/netapi/rest_tornado/test_utils.py +++ b/tests/unit/netapi/rest_tornado/test_utils.py @@ -91,7 +91,7 @@ class TestEventListener(AsyncTestCase): {'sock_dir': SOCK_DIR, 'transport': 'zeromq'}) self._finished = False # fit to event_listener's behavior - event_future = event_listener.get_event(self, 'evt1', self.stop) # get an event future + event_future = event_listener.get_event(self, 'evt1', callback=self.stop) # get an event future me.fire_event({'data': 'foo2'}, 'evt2') # fire an event we don't want me.fire_event({'data': 'foo1'}, 'evt1') # fire an event we do want self.wait() # wait for the future diff --git a/tests/unit/states/test_user.py b/tests/unit/states/test_user.py index 29d21645ef..a295ef656f 100644 --- a/tests/unit/states/test_user.py +++ b/tests/unit/states/test_user.py @@ -10,6 +10,7 @@ from __future__ import absolute_import, print_function, unicode_literals from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import TestCase, skipIf from tests.support.mock import ( + Mock, MagicMock, patch, NO_MOCK, @@ -79,6 +80,110 @@ class UserTestCase(TestCase, LoaderModuleMockMixin): ' user salt', 'result': False}) self.assertDictEqual(user.present('salt'), ret) + def test_present_invalid_uid_change(self): + mock_info = MagicMock(side_effect=[ + {'uid': 5000, + 'gid': 5000, + 'groups': ['foo'], + 'home': '/home/foo', + 'fullname': 'Foo Bar'} + ]) + dunder_salt = {'user.info': mock_info, + 'file.group_to_gid': MagicMock(side_effect=['foo']), + 'file.gid_to_group': MagicMock(side_effect=[5000])} + # side_effect used because these mocks should only be called once + with patch.dict(user.__grains__, {'kernel': 'Linux'}), \ + patch.dict(user.__salt__, dunder_salt): + ret = user.present('foo', uid=5001) + # State should have failed + self.assertFalse(ret['result']) + # Only one of uid/gid should have been flagged in the comment + self.assertEqual(ret['comment'].count('not permitted'), 1) + + def test_present_invalid_gid_change(self): + mock_info = MagicMock(side_effect=[ + {'uid': 5000, + 'gid': 5000, + 'groups': ['foo'], + 'home': '/home/foo', + 'fullname': 'Foo Bar'} + ]) + dunder_salt = {'user.info': mock_info, + 'file.group_to_gid': MagicMock(side_effect=['foo']), + 'file.gid_to_group': MagicMock(side_effect=[5000])} + # side_effect used because these mocks should only be called once + with patch.dict(user.__grains__, {'kernel': 'Linux'}), \ + patch.dict(user.__salt__, dunder_salt): + ret = user.present('foo', gid=5001) + # State should have failed + self.assertFalse(ret['result']) + # Only one of uid/gid should have been flagged in the comment + self.assertEqual(ret['comment'].count('not permitted'), 1) + + def test_present_invalid_uid_gid_change(self): + mock_info = MagicMock(side_effect=[ + {'uid': 5000, + 'gid': 5000, + 'groups': ['foo'], + 'home': '/home/foo', + 'fullname': 'Foo Bar'} + ]) + dunder_salt = {'user.info': mock_info, + 'file.group_to_gid': MagicMock(side_effect=['foo']), + 'file.gid_to_group': MagicMock(side_effect=[5000])} + # side_effect used because these mocks should only be called once + with patch.dict(user.__grains__, {'kernel': 'Linux'}), \ + patch.dict(user.__salt__, dunder_salt): + ret = user.present('foo', uid=5001, gid=5001) + # State should have failed + self.assertFalse(ret['result']) + # Both the uid and gid should have been flagged in the comment + self.assertEqual(ret['comment'].count('not permitted'), 2) + + def test_present_uid_gid_change(self): + before = {'uid': 5000, + 'gid': 5000, + 'groups': ['foo'], + 'home': '/home/foo', + 'fullname': 'Foo Bar'} + after = {'uid': 5001, + 'gid': 5001, + 'groups': ['othergroup'], + 'home': '/home/foo', + 'fullname': 'Foo Bar'} + # user.info should be called 4 times. Once the first time that + # _changes() is called, once before and after changes are applied (to + # get the before/after for the changes dict, and one last time to + # confirm that no changes still need to be made. + mock_info = MagicMock(side_effect=[before, before, after, after]) + mock_group_to_gid = MagicMock(side_effect=['foo', 'othergroup']) + mock_gid_to_group = MagicMock(side_effect=[5000, 5001]) + dunder_salt = {'user.info': mock_info, + 'user.chuid': Mock(), + 'user.chgid': Mock(), + 'file.group_to_gid': mock_group_to_gid, + 'file.gid_to_group': mock_gid_to_group} + # side_effect used because these mocks should only be called once + with patch.dict(user.__grains__, {'kernel': 'Linux'}), \ + patch.dict(user.__salt__, dunder_salt), \ + patch.dict(user.__opts__, {'test': False}), \ + patch('os.path.isdir', MagicMock(return_value=True)): + ret = user.present( + 'foo', + uid=5001, + gid=5001, + allow_uid_change=True, + allow_gid_change=True) + self.assertEqual( + ret, + {'comment': 'Updated user foo', + 'changes': {'gid': 5001, + 'uid': 5001, + 'groups': ['othergroup']}, + 'name': 'foo', + 'result': True} + ) + def test_absent(self): ''' Test to ensure that the named user is absent diff --git a/tests/unit/states/test_win_update.py b/tests/unit/states/test_win_update.py deleted file mode 100644 index 094fe6bb9b..0000000000 --- a/tests/unit/states/test_win_update.py +++ /dev/null @@ -1,133 +0,0 @@ -# -*- coding: utf-8 -*- -''' - :codeauthor: :email:`Rahul Handay <rahulha@saltstack.com>` -''' - -# Import Python Libs -from __future__ import absolute_import, unicode_literals, print_function - -# Import Salt Testing Libs -from tests.support.mixins import LoaderModuleMockMixin -from tests.support.unit import TestCase, skipIf -from tests.support.mock import ( - MagicMock, - patch, - NO_MOCK, - NO_MOCK_REASON -) - -# Import Salt Libs -import salt.states.win_update as win_update - - -class MockPyWinUpdater(object): - ''' - Mock PyWinUpdater class - ''' - def __init__(self): - pass - - @staticmethod - def SetCategories(arg): - ''' - Mock SetCategories - ''' - return arg - - @staticmethod - def SetIncludes(arg): - ''' - Mock SetIncludes - ''' - return arg - - @staticmethod - def GetInstallationResults(): - ''' - Mock GetInstallationResults - ''' - return True - - @staticmethod - def GetDownloadResults(): - ''' - Mock GetDownloadResults - ''' - return True - - @staticmethod - def SetSkips(arg): - return True - - -@skipIf(NO_MOCK, NO_MOCK_REASON) -class WinUpdateTestCase(TestCase, LoaderModuleMockMixin): - ''' - Validate the win_update state - ''' - def setup_loader_modules(self): - return {win_update: {'PyWinUpdater': MockPyWinUpdater}} - - def test_installed(self): - ''' - Test to install specified windows updates - ''' - ret = {'name': 'salt', - 'changes': {}, - 'result': False, - 'comment': '', - 'warnings': ["The 'win_update' module is deprecated, and will " - "be removed in Salt Fluorine. Please use the " - "'win_wua' module instead."]} - - mock = MagicMock(side_effect=[['Saltstack', False, 5], - ['Saltstack', True, 5], - ['Saltstack', True, 5], - ['Saltstack', True, 5]]) - with patch.object(win_update, '_search', mock): - ret.update({'comment': 'Saltstack'}) - self.assertDictEqual(win_update.installed('salt'), ret) - - mock = MagicMock(side_effect=[['dude', False, 5], - ['dude', True, 5], - ['dude', True, 5]]) - with patch.object(win_update, '_download', mock): - ret.update({'comment': 'Saltstackdude'}) - self.assertDictEqual(win_update.installed('salt'), ret) - - mock = MagicMock(side_effect=[['@Me', False, 5], - ['@Me', True, 5]]) - with patch.object(win_update, '_install', mock): - ret.update({'comment': 'Saltstackdude@Me'}) - self.assertDictEqual(win_update.installed('salt'), ret) - - ret.update({'changes': True, 'result': True}) - self.assertDictEqual(win_update.installed('salt'), ret) - - def test_downloaded(self): - ''' - Test to cache updates for later install. - ''' - ret = {'name': 'salt', - 'changes': {}, - 'result': False, - 'comment': '', - 'warnings': ["The 'win_update' module is deprecated, and will " - "be removed in Salt Fluorine. Please use the " - "'win_wua' module instead."]} - - mock = MagicMock(side_effect=[['Saltstack', False, 5], - ['Saltstack', True, 5], - ['Saltstack', True, 5]]) - with patch.object(win_update, '_search', mock): - ret.update({'comment': 'Saltstack'}) - self.assertDictEqual(win_update.downloaded('salt'), ret) - - mock = MagicMock(side_effect=[['dude', False, 5], - ['dude', True, 5]]) - with patch.object(win_update, '_download', mock): - ret.update({'comment': 'Saltstackdude'}) - self.assertDictEqual(win_update.downloaded('salt'), ret) - - ret.update({'changes': True, 'result': True}) - self.assertDictEqual(win_update.downloaded('salt'), ret) diff --git a/tests/unit/utils/test_args.py b/tests/unit/utils/test_args.py index c58e2c8d06..247c319b27 100644 --- a/tests/unit/utils/test_args.py +++ b/tests/unit/utils/test_args.py @@ -3,6 +3,7 @@ # Import python libs from __future__ import absolute_import, print_function, unicode_literals from collections import namedtuple +import logging # Import Salt Libs from salt.exceptions import SaltInvocationError @@ -19,6 +20,8 @@ from tests.support.mock import ( patch ) +log = logging.getLogger(__name__) + class ArgsTestCase(TestCase): ''' @@ -202,3 +205,63 @@ class ArgsTestCase(TestCase): self.assertEqual(fun, 'amod.afunc') self.assertEqual(args, ['double " single \'', 'double " single \'']) self.assertEqual(kwargs, {'kw1': 'equal=equal', 'kw2': 'val2'}) + + def test_yamlify_arg(self): + ''' + Test that we properly yamlify CLI input. In several of the tests below + assertIs is used instead of assertEqual. This is because we want to + confirm that the return value is not a copy of the original, but the + same instance as the original. + ''' + def _yamlify_arg(item): + log.debug('Testing yamlify_arg with %r', item) + return salt.utils.args.yamlify_arg(item) + + # Make sure non-strings are just returned back + for item in (True, False, None, 123, 45.67, ['foo'], {'foo': 'bar'}): + self.assertIs(_yamlify_arg(item), item) + + # Make sure whitespace-only isn't loaded as None + for item in ('', '\t', ' '): + self.assertIs(_yamlify_arg(item), item) + + # This value would be loaded as an int (123), the underscores would be + # ignored. Test that we identify this case and return the original + # value. + item = '1_2_3' + self.assertIs(_yamlify_arg(item), item) + + # The '#' is treated as a comment when not part of a data structure, we + # don't want that behavior + for item in ('# hash at beginning', 'Hello world! # hash elsewhere'): + self.assertIs(_yamlify_arg(item), item) + + # However we _do_ want the # to be intact if it _is_ within a data + # structure. + item = '["foo", "bar", "###"]' + self.assertEqual(_yamlify_arg(item), ["foo", "bar", "###"]) + item = '{"foo": "###"}' + self.assertEqual(_yamlify_arg(item), {"foo": "###"}) + + # The string "None" should load _as_ None + self.assertIs(_yamlify_arg('None'), None) + + # Leading dashes, or strings containing colons, will result in lists + # and dicts, and we only want to load lists and dicts when the strings + # look like data structures. + for item in ('- foo', 'foo: bar'): + self.assertIs(_yamlify_arg(item), item) + + # Make sure we don't load '|' as '' + item = '|' + self.assertIs(_yamlify_arg(item), item) + + # Make sure we load ints, floats, and strings correctly + self.assertEqual(_yamlify_arg('123'), 123) + self.assertEqual(_yamlify_arg('45.67'), 45.67) + self.assertEqual(_yamlify_arg('foo'), 'foo') + + # We tested list/dict loading above, but there is separate logic when + # the string contains a '#', so we need to test again here. + self.assertEqual(_yamlify_arg('["foo", "bar"]'), ["foo", "bar"]) + self.assertEqual(_yamlify_arg('{"foo": "bar"}'), {"foo": "bar"})